==================================================================================================== ========================================= Pod describe ========================================= ==================================================================================================== Name: lodemon-79cd4858c7-2r8x4 Namespace: xlou Priority: 0 Service Account: k8s-svc-acct-lodemon Node: gke-xlou-cdm-ds-0f601751-mc5d/10.142.0.43 Start Time: Fri, 19 Apr 2024 15:58:00 +0000 Labels: app=lodemon app.kubernetes.io/name=lodemon pod-template-hash=79cd4858c7 skaffold.dev/run-id=61982daa-7a04-4620-aa6e-58ccd9d67703 Annotations: Status: Running IP: 10.106.46.167 IPs: IP: 10.106.46.167 Controlled By: ReplicaSet/lodemon-79cd4858c7 Containers: lodemon: Container ID: containerd://118efbbe9c5b3289f4e9f3ac6454f1f978f9e997ca166dbb0ccfb711a3bb8a51 Image: gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable Image ID: gcr.io/engineeringpit/lodestar-images/lodestarbox@sha256:03f76e6326d713f4d73dc474e0e25fbbbaecc7c1d8c0590bfd94d68ac0d98c97 Port: 8080/TCP Host Port: 0/TCP Command: python3 Args: /lodestar/scripts/lodemon_run.py -W default State: Running Started: Fri, 19 Apr 2024 15:58:00 +0000 Ready: True Restart Count: 0 Limits: cpu: 2 memory: 2Gi Requests: cpu: 1 memory: 1Gi Liveness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Readiness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Environment: SKAFFOLD_PROFILE: medium Mounts: /lodestar/config/config.yaml from config (rw,path="config.yaml") /lodestar/config/deployments_to_monitor.yaml from deployments-config (rw,path="deployments_to_monitor.yaml") /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-ts6n5 (ro) Conditions: Type Status Initialized True Ready True ContainersReady True PodScheduled True Volumes: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-config Optional: false deployments-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-deployments Optional: false kube-api-access-ts6n5: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: ==================================================================================================== =========================================== Pod logs =========================================== ==================================================================================================== * Serving Flask app 'lodemon_run' * Debug mode: off WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. * Running on all addresses (0.0.0.0) * Running on http://127.0.0.1:8080 * Running on http://10.106.46.167:8080 Press CTRL+C to quit 16:58:57 INFO 16:58:57 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:57 INFO [loop_until]: OK (rc = 0) 16:58:57 DEBUG --- stdout --- 16:58:57 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:57 DEBUG --- stderr --- 16:58:57 DEBUG 16:58:57 INFO 16:58:57 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:57 INFO [loop_until]: OK (rc = 0) 16:58:57 DEBUG --- stdout --- 16:58:57 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:57 DEBUG --- stderr --- 16:58:57 DEBUG 16:58:57 INFO 16:58:57 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:57 INFO [loop_until]: OK (rc = 0) 16:58:57 DEBUG --- stdout --- 16:58:57 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:57 DEBUG --- stderr --- 16:58:57 DEBUG 16:58:57 INFO 16:58:57 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:57 INFO [loop_until]: OK (rc = 0) 16:58:57 DEBUG --- stdout --- 16:58:57 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:57 DEBUG --- stderr --- 16:58:57 DEBUG 16:58:57 INFO 16:58:57 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:57 INFO [loop_until]: OK (rc = 0) 16:58:57 DEBUG --- stdout --- 16:58:57 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:57 DEBUG --- stderr --- 16:58:57 DEBUG 16:58:57 INFO 16:58:57 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:57 INFO [loop_until]: OK (rc = 0) 16:58:57 DEBUG --- stdout --- 16:58:57 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:57 DEBUG --- stderr --- 16:58:57 DEBUG 16:58:58 INFO 16:58:58 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:58 INFO [loop_until]: OK (rc = 0) 16:58:58 DEBUG --- stdout --- 16:58:58 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:58 DEBUG --- stderr --- 16:58:58 DEBUG 16:58:58 INFO 16:58:58 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:58 INFO [loop_until]: OK (rc = 0) 16:58:58 DEBUG --- stdout --- 16:58:58 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:58 DEBUG --- stderr --- 16:58:58 DEBUG 16:58:58 INFO 16:58:58 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:58 INFO [loop_until]: OK (rc = 0) 16:58:58 DEBUG --- stdout --- 16:58:58 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:58 DEBUG --- stderr --- 16:58:58 DEBUG 16:58:58 INFO 16:58:58 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:58 INFO [loop_until]: OK (rc = 0) 16:58:58 DEBUG --- stdout --- 16:58:58 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:58 DEBUG --- stderr --- 16:58:58 DEBUG 16:58:58 INFO 16:58:58 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:58 INFO [loop_until]: OK (rc = 0) 16:58:58 DEBUG --- stdout --- 16:58:58 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:58 DEBUG --- stderr --- 16:58:58 DEBUG 16:58:58 INFO 16:58:58 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:58 INFO [loop_until]: OK (rc = 0) 16:58:58 DEBUG --- stdout --- 16:58:58 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:58 DEBUG --- stderr --- 16:58:58 DEBUG 16:58:58 INFO 16:58:58 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:58 INFO [loop_until]: OK (rc = 0) 16:58:58 DEBUG --- stdout --- 16:58:58 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:58 DEBUG --- stderr --- 16:58:58 DEBUG 16:58:59 INFO 16:58:59 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:59 INFO [loop_until]: OK (rc = 0) 16:58:59 DEBUG --- stdout --- 16:58:59 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:59 DEBUG --- stderr --- 16:58:59 DEBUG 16:58:59 INFO 16:58:59 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:59 INFO [loop_until]: OK (rc = 0) 16:58:59 DEBUG --- stdout --- 16:58:59 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:59 DEBUG --- stderr --- 16:58:59 DEBUG 16:58:59 INFO 16:58:59 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:59 INFO [loop_until]: OK (rc = 0) 16:58:59 DEBUG --- stdout --- 16:58:59 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:59 DEBUG --- stderr --- 16:58:59 DEBUG 16:58:59 INFO 16:58:59 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:59 INFO [loop_until]: OK (rc = 0) 16:58:59 DEBUG --- stdout --- 16:58:59 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:59 DEBUG --- stderr --- 16:58:59 DEBUG 16:58:59 INFO 16:58:59 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:59 INFO [loop_until]: OK (rc = 0) 16:58:59 DEBUG --- stdout --- 16:58:59 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:59 DEBUG --- stderr --- 16:58:59 DEBUG 16:58:59 INFO 16:58:59 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:59 INFO [loop_until]: OK (rc = 0) 16:58:59 DEBUG --- stdout --- 16:58:59 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:59 DEBUG --- stderr --- 16:58:59 DEBUG 16:58:59 INFO 16:58:59 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:58:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:58:59 INFO [loop_until]: OK (rc = 0) 16:58:59 DEBUG --- stdout --- 16:58:59 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:58:59 DEBUG --- stderr --- 16:58:59 DEBUG 16:59:00 INFO 16:59:00 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:59:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:59:00 INFO [loop_until]: OK (rc = 0) 16:59:00 DEBUG --- stdout --- 16:59:00 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:59:00 DEBUG --- stderr --- 16:59:00 DEBUG 16:59:00 INFO 16:59:00 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:59:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:59:00 INFO [loop_until]: OK (rc = 0) 16:59:00 DEBUG --- stdout --- 16:59:00 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:59:00 DEBUG --- stderr --- 16:59:00 DEBUG 16:59:00 INFO 16:59:00 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:59:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:59:00 INFO [loop_until]: OK (rc = 0) 16:59:00 DEBUG --- stdout --- 16:59:00 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:59:00 DEBUG --- stderr --- 16:59:00 DEBUG 16:59:00 INFO 16:59:00 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:59:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:59:00 INFO [loop_until]: OK (rc = 0) 16:59:00 DEBUG --- stdout --- 16:59:00 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:59:00 DEBUG --- stderr --- 16:59:00 DEBUG 16:59:00 INFO 16:59:00 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:59:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:59:00 INFO [loop_until]: OK (rc = 0) 16:59:00 DEBUG --- stdout --- 16:59:00 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:59:00 DEBUG --- stderr --- 16:59:00 DEBUG 16:59:00 INFO 16:59:00 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:59:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:59:00 INFO [loop_until]: OK (rc = 0) 16:59:00 DEBUG --- stdout --- 16:59:00 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:59:00 DEBUG --- stderr --- 16:59:00 DEBUG 16:59:00 INFO 16:59:00 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 16:59:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:59:00 INFO [loop_until]: OK (rc = 0) 16:59:00 DEBUG --- stdout --- 16:59:00 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 16:59:00 DEBUG --- stderr --- 16:59:00 DEBUG 16:59:00 INFO Initializing monitoring instance threads 16:59:00 DEBUG Monitoring instance thread list: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 16:59:00 INFO Starting instance threads 16:59:00 INFO 16:59:00 INFO Thread started 16:59:00 INFO [loop_until]: kubectl --namespace=xlou top node 16:59:00 INFO 16:59:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:59:00 INFO Thread started 16:59:00 INFO [loop_until]: kubectl --namespace=xlou top pods 16:59:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 DEBUG --- status code --- 16:59:00 INFO Thread started 16:59:00 DEBUG --- status code --- 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 DEBUG --- status code --- 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 DEBUG --- status code --- 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 INFO Thread started 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG --- status code --- 16:59:00 INFO [loop_until]: OK (rc = 0) 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG --- http response --- 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 DEBUG --- http response --- 16:59:00 INFO Thread started 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 DEBUG --- stdout --- 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 INFO Thread started 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 8m 1072Mi am-6fd554f8f7-btd4s 7m 3082Mi am-6fd554f8f7-wxcsj 11m 3243Mi ds-cts-0 5m 11345Mi ds-cts-1 5m 11310Mi ds-cts-2 5m 10318Mi ds-idrepo-0 9m 3431Mi ds-idrepo-1 9m 3293Mi ds-idrepo-2 8m 3290Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 1m 54Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1m 312Mi 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG --- http response --- 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 DEBUG --- http response --- 16:59:00 INFO Thread started 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG --- stderr --- 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 DEBUG ----- output ----- 16:59:00 INFO Thread started 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG ----- output ----- 16:59:00 INFO [loop_until]: OK (rc = 0) 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG { 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG ----- output ----- 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 DEBUG { 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG { 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713542340" 16:59:00 DEBUG { 16:59:00 INFO Thread started 16:59:00 DEBUG --- stdout --- 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 16:59:00 DEBUG { 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "data": { 16:59:00 DEBUG "data": { 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG { 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG "status": "success", 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542340" 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO Thread started 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 77m 0% 6173Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 67m 0% 6977Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 43m 0% 2685Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 4406Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 60m 0% 14318Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5224Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 5953Mi 5% gke-xlou-cdm-ds-0f601751-cf7b 77m 0% 3221Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 61m 0% 5430Mi 4% gke-xlou-cdm-ds-0f601751-j488 61m 0% 6482Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 66m 0% 16086Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 90m 0% 7061Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 59m 0% 12739Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 71m 0% 4152Mi 7% 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "data": { 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG "data": { 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "data": { 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 16:59:00 DEBUG "data": { 16:59:00 DEBUG --- status code --- 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542340" 16:59:00 DEBUG { 16:59:00 INFO Thread started 16:59:00 DEBUG --- stderr --- 16:59:00 DEBUG "data": { 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG "data": { 16:59:00 DEBUG "data": { 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG "data": { 16:59:00 DEBUG "data": { 16:59:00 DEBUG "data": { 16:59:00 DEBUG "data": { 16:59:00 DEBUG "data": { 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG "resultType": "vector", 16:59:00 ERROR [http_cmd]: ERROR 16:59:00 DEBUG "result": [ 16:59:00 DEBUG { 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 16:59:00 DEBUG "data": { 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG 16:59:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542340" 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG "result": [ 16:59:00 DEBUG "result": [ 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG "data": { 16:59:00 DEBUG "result": [ 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 16:59:00 DEBUG "result": [ 16:59:00 ERROR --- status code --- 16:59:00 DEBUG "result": [ 16:59:00 DEBUG "metric": { 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG "result": [ 16:59:00 DEBUG { 16:59:00 DEBUG "result": [ 16:59:00 DEBUG { 16:59:00 DEBUG "metric": { 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG "data": { 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG "result": [ 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG { 16:59:00 DEBUG "result": [ 16:59:00 DEBUG "result": [ 16:59:00 DEBUG "result": [ 16:59:00 DEBUG "result": [ 16:59:00 DEBUG "result": [ 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG { 16:59:00 ERROR http status code is 400 (expected [200, 401, 403, 404]) 16:59:00 DEBUG { 16:59:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 16:59:00 DEBUG { 16:59:00 DEBUG "result": [ 16:59:00 DEBUG { 16:59:00 DEBUG "status": "success", 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 DEBUG "pod": "ds-idrepo-0" 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 INFO Thread started 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 16:59:00 DEBUG "metric": { 16:59:00 DEBUG "metric": { 16:59:00 DEBUG { 16:59:00 DEBUG "metric": { 16:59:00 DEBUG "metric": { 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG { 16:59:00 DEBUG "metric": { 16:59:00 ERROR --- http response --- 16:59:00 DEBUG "metric": { 16:59:00 DEBUG }, 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG { 16:59:00 DEBUG "metric": { 16:59:00 DEBUG "data": { 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG }, 16:59:00 DEBUG --- http response --- 16:59:00 INFO All threads has been started 127.0.0.1 - - [19/Apr/2024 16:59:00] "GET /monitoring/start HTTP/1.1" 200 - 16:59:00 DEBUG "result": [ 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG { 16:59:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 16:59:00 DEBUG "result": [ 16:59:00 DEBUG "instance": "10.142.0.107:9100" 16:59:00 DEBUG "pod": "ds-idrepo-0" 16:59:00 DEBUG "metric": { 16:59:00 DEBUG "metric": { 16:59:00 DEBUG "metric": { 16:59:00 DEBUG "metric": { 16:59:00 DEBUG "metric": { 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "instance": "10.142.0.107:9100" 16:59:00 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 16:59:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 16:59:00 DEBUG "value": [ 16:59:00 DEBUG "data": { 16:59:00 DEBUG "metric": { 16:59:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 16:59:00 DEBUG { 16:59:00 DEBUG "pod": "am-6fd554f8f7-2pw5f" 16:59:00 DEBUG "status": "success", 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 INFO [http_cmd]: http status code OK 16:59:00 DEBUG }, 16:59:00 DEBUG { 16:59:00 DEBUG }, 16:59:00 DEBUG }, 16:59:00 DEBUG "value": [ 16:59:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 16:59:00 DEBUG "pod": "ds-cts-0" 16:59:00 DEBUG "pod": "am-6fd554f8f7-2pw5f" 16:59:00 DEBUG "pod": "ds-idrepo-0" 16:59:00 DEBUG "data": { 16:59:00 DEBUG }, 16:59:00 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 16:59:00 DEBUG }, 16:59:00 DEBUG 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG "pod": "am-6fd554f8f7-2pw5f" 16:59:00 DEBUG }, 16:59:00 DEBUG "result": [ 16:59:00 DEBUG "value": [ 16:59:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG "metric": { 16:59:00 DEBUG }, 16:59:00 DEBUG "data": { 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG --- status code --- 16:59:00 DEBUG "value": [ 16:59:00 DEBUG 16:59:00 DEBUG "metric": { 16:59:00 DEBUG "instance": "10.142.0.107:9100" 16:59:00 DEBUG 16:59:00 DEBUG }, 16:59:00 DEBUG }, 16:59:00 DEBUG }, 16:59:00 DEBUG }, 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG "value": [ 16:59:00 DEBUG "value": [ 16:59:00 DEBUG 16:59:00 DEBUG "result": [ 16:59:00 DEBUG }, 16:59:00 DEBUG "value": [ 16:59:00 DEBUG { 16:59:00 DEBUG 16:59:00 DEBUG ----- output ----- 16:59:00 DEBUG { 16:59:00 DEBUG "status": "success", 16:59:00 DEBUG "pod": "am-6fd554f8f7-2pw5f" 16:59:00 DEBUG "resultType": "vector", 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 16:59:00 DEBUG "value": [ 16:59:00 DEBUG 16:59:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 16:59:00 DEBUG }, 16:59:00 DEBUG }, 16:59:00 DEBUG "value": [ 16:59:00 DEBUG "value": [ 16:59:00 DEBUG "value": [ 16:59:00 DEBUG "value": [ 16:59:00 DEBUG "result": [ 16:59:00 DEBUG 16:59:00 DEBUG 16:59:00 DEBUG 16:59:00 DEBUG { 16:59:00 DEBUG "value": [ 16:59:00 DEBUG 16:59:00 DEBUG "metric": { 16:59:00 DEBUG 16:59:00 DEBUG { 16:59:00 DEBUG "value": [ 16:59:00 DEBUG "data": { 16:59:01 DEBUG "resultType": "vector", 16:59:00 DEBUG "result": [ 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG --- http response --- 16:59:00 DEBUG 16:59:00 DEBUG 16:59:00 DEBUG 16:59:00 DEBUG "value": [ 16:59:00 DEBUG "value": [ 16:59:00 DEBUG 16:59:00 DEBUG 16:59:00 DEBUG 16:59:00 DEBUG 16:59:00 DEBUG { 16:59:00 DEBUG 16:59:00 DEBUG 16:59:00 DEBUG [...] 16:59:01 DEBUG "metric": { 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG "pod": "ds-cts-0" 16:59:01 DEBUG 16:59:01 DEBUG "status": "success", 16:59:01 DEBUG 16:59:01 DEBUG "result": [ 16:59:00 DEBUG }, 16:59:01 DEBUG { 16:59:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 16:59:01 DEBUG 16:59:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 16:59:01 DEBUG [...] 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG "metric": { 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG "pod": "ds-cts-0" 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG }, 16:59:01 DEBUG [...] 16:59:01 DEBUG "data": { 16:59:01 DEBUG "resultType": "vector", 16:59:01 DEBUG { 16:59:01 DEBUG "value": [ 16:59:01 DEBUG "metric": { 16:59:01 DEBUG ----- output ----- 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG [...] 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 16:59:01 DEBUG [...] 16:59:01 DEBUG [...] 16:59:01 DEBUG 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG [...] 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG "result": [ 16:59:01 DEBUG "metric": { 16:59:01 DEBUG 16:59:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 16:59:01 DEBUG { 16:59:01 DEBUG [...] 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG [...] 16:59:01 DEBUG [...] 16:59:01 DEBUG [...] 16:59:01 DEBUG [...] 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG "value": [ 16:59:01 DEBUG [...] 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG { 16:59:01 DEBUG "pod": "ds-idrepo-0" 16:59:01 DEBUG 16:59:01 DEBUG }, 16:59:01 DEBUG "status": "success", 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG [...] 16:59:01 DEBUG [...] 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG [...] 16:59:01 DEBUG "metric": { 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG "value": [ 16:59:01 DEBUG "data": { 16:59:01 DEBUG 16:59:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 16:59:01 DEBUG 16:59:01 DEBUG "pod": "ds-idrepo-2" 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 16:59:01 DEBUG "pod": "ds-idrepo-2" 16:59:01 DEBUG 16:59:01 DEBUG "instance": "10.142.0.107:9100" 16:59:01 DEBUG "value": [ 16:59:01 DEBUG [...] 16:59:01 DEBUG 16:59:01 DEBUG "resultType": "vector", 16:59:01 DEBUG 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG "pod": "ds-cts-2" 16:59:01 DEBUG "value": [ 16:59:01 DEBUG "pod": "ds-idrepo-2" 16:59:01 DEBUG "instance": "10.142.0.43:9100" 16:59:01 DEBUG "pod": "ds-cts-0" 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG [...] 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG "result": [ 16:59:01 DEBUG "instance": "10.142.0.43:9100" 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG 16:59:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG }, 16:59:01 DEBUG }, 16:59:01 DEBUG }, 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG [...] 16:59:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG { 16:59:01 DEBUG "value": [ 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 16:59:01 DEBUG "pod": "ds-cts-0" 16:59:01 DEBUG "0" 16:59:01 DEBUG "instance": "10.142.0.43:9100" 16:59:01 DEBUG }, 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG "0" 16:59:01 DEBUG "value": [ 16:59:01 DEBUG "value": [ 16:59:01 DEBUG "0" 16:59:01 DEBUG 16:59:01 DEBUG }, 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG [...] 16:59:01 DEBUG "metric": { 16:59:01 DEBUG "instance": "10.142.0.107:9100" 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 16:59:01 DEBUG }, 16:59:01 DEBUG ] 16:59:01 DEBUG }, 16:59:01 DEBUG "value": [ 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG "0" 16:59:01 DEBUG ] 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG 16:59:01 DEBUG "0" 16:59:01 DEBUG ] 16:59:01 DEBUG }, 16:59:01 DEBUG [...] 16:59:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 16:59:01 DEBUG 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG }, 16:59:01 DEBUG "19489.9" 16:59:01 DEBUG }, 16:59:01 DEBUG "value": [ 16:59:01 DEBUG } 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG "0" 16:59:01 DEBUG 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG ] 16:59:01 DEBUG 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG "0" 16:59:01 DEBUG "pod": "ds-cts-2" 16:59:01 DEBUG 16:59:01 DEBUG } 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG "424128512" 16:59:01 DEBUG "value": [ 16:59:01 DEBUG ] 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG ] 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG ] 16:59:01 DEBUG [...] 16:59:01 DEBUG "13287.033333333333" 16:59:01 DEBUG } 16:59:01 DEBUG 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG "0" 16:59:01 DEBUG ] 16:59:01 DEBUG }, 16:59:01 DEBUG 16:59:01 DEBUG ] 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 16:59:01 DEBUG ] 16:59:01 DEBUG 16:59:01 DEBUG } 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG "0" 16:59:01 DEBUG } 16:59:01 DEBUG "0.004000000000002047" 16:59:01 DEBUG "0" 16:59:01 DEBUG } 16:59:01 DEBUG 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG "pod": "ds-cts-2" 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG } 16:59:01 DEBUG "0" 16:59:01 DEBUG 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 16:59:01 DEBUG } 16:59:01 DEBUG 16:59:01 DEBUG ] 16:59:01 DEBUG "0.028165348351214987" 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG 16:59:01 DEBUG } 16:59:01 DEBUG }, 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG [...] 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG "pod": "ds-cts-2" 16:59:01 DEBUG "0" 16:59:01 DEBUG }, 16:59:01 DEBUG ] 16:59:01 DEBUG 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG ] 16:59:01 DEBUG 16:59:01 DEBUG ] 16:59:01 DEBUG "value": [ 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG "0" 16:59:01 DEBUG } 16:59:01 DEBUG 16:59:01 DEBUG } 16:59:01 DEBUG }, 16:59:01 DEBUG ] 16:59:01 DEBUG "value": [ 16:59:01 DEBUG } 16:59:01 DEBUG [...] 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 16:59:01 DEBUG } 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG 16:59:01 DEBUG 16:59:01 DEBUG "value": [ 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG } 16:59:01 DEBUG 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG }, 16:59:01 DEBUG } 16:59:01 DEBUG "0" 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG "instance": "10.142.0.43:9100" 16:59:01 DEBUG } 16:59:01 DEBUG "0" 16:59:01 DEBUG "0.006" 16:59:01 DEBUG 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG "value": [ 16:59:01 DEBUG ] 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG }, 16:59:01 DEBUG ] 16:59:01 DEBUG ] 16:59:01 DEBUG ] 16:59:01 DEBUG 16:59:01 DEBUG } 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG "value": [ 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG "instance": "10.142.0.43:9100" 16:59:01 DEBUG "2" 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG "1627416058880" 16:59:01 DEBUG }, 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG "value": [ 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG 1713542340, 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG "0.005800000000095906" 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG } 16:59:01 DEBUG ] 16:59:01 DEBUG } 16:59:01 DEBUG } 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO 17:00:01 INFO [loop_until]: kubectl --namespace=xlou top pods 17:00:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:00:01 INFO 17:00:01 INFO [loop_until]: kubectl --namespace=xlou top node 17:00:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713542401" 17:00:01 ERROR [http_cmd]: ERROR 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713542401" 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542401" 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [loop_until]: OK (rc = 0) 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 ERROR --- status code --- 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- stdout --- 17:00:01 DEBUG --- status code --- 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 DEBUG --- status code --- 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 DEBUG --- status code --- 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 4713m 3755Mi am-6fd554f8f7-btd4s 6641m 3447Mi am-6fd554f8f7-wxcsj 7266m 3877Mi ds-cts-0 2911m 11370Mi ds-cts-1 1424m 11340Mi ds-cts-2 2561m 10338Mi ds-idrepo-0 2407m 3424Mi ds-idrepo-1 2031m 3293Mi ds-idrepo-2 2141m 3291Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 61Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 3436m 750Mi 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 INFO [http_cmd]: http status code OK 17:00:01 ERROR --- http response --- 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- stderr --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- status code --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:00:01 DEBUG 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:00:01 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG --- http response --- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG { 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG { 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:00:01 DEBUG { 17:00:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG { 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG { 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG ----- output ----- 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "data": { 17:00:01 DEBUG "data": { 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "data": { 17:00:01 DEBUG "data": { 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "data": { 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "data": { 17:00:01 DEBUG { 17:00:01 DEBUG "data": { 17:00:01 DEBUG { 17:00:01 DEBUG "data": { 17:00:01 DEBUG "data": { 17:00:01 DEBUG "data": { 17:00:01 DEBUG "data": { 17:00:01 DEBUG "data": { 17:00:01 DEBUG "data": { 17:00:01 DEBUG "data": { 17:00:01 DEBUG "data": { 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "data": { 17:00:01 DEBUG "data": { 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "data": { 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "data": { 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "data": { 17:00:01 DEBUG "data": { 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "status": "success", 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "data": { 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "data": { 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "data": { 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG { 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG { 17:00:01 DEBUG "result": [ 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG "result": [ 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG "result": [ 17:00:01 DEBUG { 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "result": [ 17:00:01 DEBUG { 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG { 17:00:01 DEBUG "resultType": "vector", 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "metric": { 17:00:01 DEBUG { 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:00:01 DEBUG "metric": { 17:00:01 DEBUG { 17:00:01 DEBUG { 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "result": [ 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "metric": { 17:00:01 DEBUG { 17:00:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:00:01 DEBUG { 17:00:01 DEBUG "instance": "10.142.0.107:9100" 17:00:01 DEBUG { 17:00:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:00:01 DEBUG "pod": "ds-cts-0" 17:00:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "pod": "ds-idrepo-0" 17:00:01 DEBUG }, 17:00:01 DEBUG { 17:00:01 DEBUG "instance": "10.142.0.107:9100" 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "instance": "10.142.0.107:9100" 17:00:01 DEBUG { 17:00:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:00:01 DEBUG { 17:00:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:00:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:00:01 DEBUG "pod": "ds-idrepo-0" 17:00:01 DEBUG "pod": "ds-idrepo-0" 17:00:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:00:01 DEBUG "metric": { 17:00:01 DEBUG }, 17:00:01 DEBUG "metric": { 17:00:01 DEBUG }, 17:00:01 DEBUG "metric": { 17:00:01 DEBUG }, 17:00:01 DEBUG }, 17:00:01 DEBUG }, 17:00:01 DEBUG "pod": "ds-idrepo-0" 17:00:01 DEBUG }, 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "metric": { 17:00:01 DEBUG }, 17:00:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:00:01 DEBUG "instance": "10.142.0.107:9100" 17:00:01 DEBUG }, 17:00:01 DEBUG "metric": { 17:00:01 DEBUG }, 17:00:01 INFO [loop_until]: OK (rc = 0) 17:00:01 DEBUG "metric": { 17:00:01 DEBUG "instance": "10.142.0.107:9100" 17:00:01 DEBUG }, 17:00:01 DEBUG }, 17:00:01 DEBUG }, 17:00:01 DEBUG }, 17:00:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "pod": "ds-cts-0" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 17:00:01 DEBUG }, 17:00:01 DEBUG "pod": "ds-cts-0" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG }, 17:00:01 DEBUG }, 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG --- stdout --- 17:00:01 DEBUG }, 17:00:01 DEBUG }, 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "value": [ 17:00:01 DEBUG }, 17:00:01 DEBUG 17:00:01 DEBUG }, 17:00:01 DEBUG 17:00:01 DEBUG }, 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG "value": [ 17:00:01 DEBUG }, 17:00:01 DEBUG 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 17:00:01 DEBUG }, 17:00:01 DEBUG 17:00:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6178Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 2066m 6% 6988Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 47m 0% 2686Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 5823m 19% 7017Mi 6% gke-xlou-cdm-default-pool-1b0769c8-w15q 2354m 7% 14365Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 136m 0% 5228Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 8069m 27% 6716Mi 6% gke-xlou-cdm-ds-0f601751-cf7b 6731m 22% 2995Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 5450m 18% 5872Mi 5% gke-xlou-cdm-ds-0f601751-j488 2643m 8% 6503Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 2906m 9% 16102Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 2316m 7% 7090Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 2297m 7% 12777Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 4789m 30% 4592Mi 7% 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 17:00:01 DEBUG --- stderr --- 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG [...] 17:00:01 DEBUG [...] 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG [...] 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:00:01 DEBUG 17:00:01 DEBUG "pod": "ds-idrepo-2" 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:00:01 DEBUG 17:00:01 DEBUG "instance": "10.142.0.43:9100" 17:00:01 DEBUG 17:00:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:00:01 DEBUG }, 17:00:01 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:00:01 DEBUG "pod": "ds-cts-2" 17:00:01 DEBUG }, 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG "instance": "10.142.0.43:9100" 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG "instance": "10.142.0.43:9100" 17:00:01 DEBUG 17:00:01 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:00:01 DEBUG 17:00:01 DEBUG 17:00:01 DEBUG }, 17:00:01 DEBUG 17:00:01 DEBUG }, 17:00:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:00:01 DEBUG }, 17:00:01 DEBUG }, 17:00:01 DEBUG 17:00:01 DEBUG }, 17:00:01 DEBUG 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "pod": "ds-cts-2" 17:00:01 DEBUG }, 17:00:01 DEBUG }, 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "pod": "ds-idrepo-2" 17:00:01 DEBUG 17:00:01 DEBUG }, 17:00:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:00:01 DEBUG "instance": "10.142.0.43:9100" 17:00:01 DEBUG }, 17:00:01 DEBUG 17:00:01 DEBUG }, 17:00:01 DEBUG 17:00:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "pod": "ds-idrepo-2" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "pod": "ds-cts-2" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG }, 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG }, 17:00:01 DEBUG "pod": "ds-cts-2" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG }, 17:00:01 DEBUG }, 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "instance": "10.142.0.43:9100" 17:00:01 DEBUG }, 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG }, 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG }, 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG }, 17:00:01 DEBUG "0" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG "0" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG }, 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG }, 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG }, 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "854888448" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "0" 17:00:01 DEBUG }, 17:00:01 DEBUG "323.3" 17:00:01 DEBUG "0" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "0.028999999999996355" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG ] 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG "0" 17:00:01 DEBUG "0" 17:00:01 DEBUG ] 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "7630406.033333333" 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG "7638628.699999999" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG "880.7649253917451" 17:00:01 DEBUG "value": [ 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG ] 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG ] 17:00:01 DEBUG "value": [ 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG ] 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG } 17:00:01 DEBUG "0" 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG "0" 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG ] 17:00:01 DEBUG "4839.5" 17:00:01 DEBUG "1627516998656" 17:00:01 DEBUG ] 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG ] 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG "279.29590444444443" 17:00:01 DEBUG } 17:00:01 DEBUG "0" 17:00:01 DEBUG } 17:00:01 DEBUG 1713542401, 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG "0" 17:00:01 DEBUG } 17:00:01 DEBUG "2.194496265071409" 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG "0" 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG "0.006" 17:00:01 DEBUG } 17:00:01 DEBUG "0.04576666666662561" 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG "279.65867777777777" 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG ] 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:00:01 DEBUG } 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:01:01 INFO 17:01:01 INFO [loop_until]: kubectl --namespace=xlou top pods 17:01:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 ERROR [http_cmd]: ERROR 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 ERROR --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 ERROR --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:01:01 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:01:01 INFO 17:01:01 INFO [loop_until]: kubectl --namespace=xlou top node 17:01:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:01:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "data": { 17:01:01 DEBUG "data": { 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "result": [ 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "pod": "ds-idrepo-0" 17:01:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:01:01 DEBUG }, 17:01:01 DEBUG }, 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG "pod": "ds-cts-2" 17:01:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:01:01 DEBUG }, 17:01:01 DEBUG }, 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG "0" 17:01:01 DEBUG "2343.233333333333" 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 DEBUG } 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [loop_until]: OK (rc = 0) 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713542461" 17:01:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713542461" 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 DEBUG --- stdout --- 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [loop_until]: OK (rc = 0) 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 DEBUG --- status code --- 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11362m 4745Mi am-6fd554f8f7-btd4s 11373m 4581Mi am-6fd554f8f7-wxcsj 12035m 4661Mi ds-cts-0 6068m 8713Mi ds-cts-1 6101m 8880Mi ds-cts-2 6392m 7717Mi ds-idrepo-0 5546m 3429Mi ds-idrepo-1 5433m 3294Mi ds-idrepo-2 5625m 3324Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-79cd4858c7-2r8x4 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7701m 806Mi 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 DEBUG --- stdout --- 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG --- stderr --- 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6182Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5447m 18% 6994Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 49m 0% 2684Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11217m 37% 8012Mi 7% gke-xlou-cdm-default-pool-1b0769c8-w15q 6205m 20% 11768Mi 10% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5187Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11453m 38% 7460Mi 6% gke-xlou-cdm-ds-0f601751-cf7b 12434m 41% 2990Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12955m 43% 6867Mi 6% gke-xlou-cdm-ds-0f601751-j488 6349m 21% 6530Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 6169m 20% 13735Mi 12% gke-xlou-cdm-ds-0f601751-nj6q 6462m 21% 7098Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 6395m 21% 10225Mi 9% gke-xlou-cdm-frontend-a8771548-i82n 8053m 50% 4626Mi 7% 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 DEBUG --- stderr --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG ----- output ----- 17:01:01 INFO [http_cmd]: http status code OK 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:01:01 DEBUG { 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG { 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG { 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG { 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG --- status code --- 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG { 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG { 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG { 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG { 17:01:01 DEBUG "data": { 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "data": { 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "data": { 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "data": { 17:01:01 DEBUG { 17:01:01 DEBUG "data": { 17:01:01 DEBUG "data": { 17:01:01 DEBUG "data": { 17:01:01 DEBUG --- http response --- 17:01:01 DEBUG "data": { 17:01:01 DEBUG "data": { 17:01:01 DEBUG "data": { 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG { 17:01:01 DEBUG "data": { 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "data": { 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "data": { 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "data": { 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "data": { 17:01:01 DEBUG "data": { 17:01:01 DEBUG "data": { 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "data": { 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "data": { 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "data": { 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "result": [ 17:01:01 DEBUG ----- output ----- 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "data": { 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG { 17:01:01 DEBUG "result": [ 17:01:01 DEBUG { 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG "result": [ 17:01:01 DEBUG { 17:01:01 DEBUG "result": [ 17:01:01 DEBUG { 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG { 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "metric": { 17:01:01 DEBUG { 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "result": [ 17:01:01 DEBUG { 17:01:01 DEBUG "metric": { 17:01:01 DEBUG { 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "status": "success", 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "metric": { 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG { 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "metric": { 17:01:01 DEBUG { 17:01:01 DEBUG "pod": "ds-idrepo-0" 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "pod": "ds-idrepo-0" 17:01:01 DEBUG { 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:01:01 DEBUG { 17:01:01 DEBUG "pod": "ds-idrepo-0" 17:01:01 DEBUG "pod": "ds-idrepo-0" 17:01:01 DEBUG "pod": "ds-idrepo-0" 17:01:01 DEBUG "instance": "10.142.0.107:9100" 17:01:01 DEBUG "data": { 17:01:01 DEBUG "instance": "10.142.0.107:9100" 17:01:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "instance": "10.142.0.107:9100" 17:01:01 DEBUG }, 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "pod": "ds-cts-0" 17:01:01 DEBUG "metric": { 17:01:01 DEBUG }, 17:01:01 DEBUG "instance": "10.142.0.107:9100" 17:01:01 DEBUG }, 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:01:01 DEBUG }, 17:01:01 DEBUG "pod": "ds-cts-0" 17:01:01 DEBUG }, 17:01:01 DEBUG "value": [ 17:01:01 DEBUG }, 17:01:01 DEBUG }, 17:01:01 DEBUG }, 17:01:01 DEBUG }, 17:01:01 DEBUG "resultType": "vector", 17:01:01 DEBUG }, 17:01:01 DEBUG }, 17:01:01 DEBUG "metric": { 17:01:01 DEBUG "value": [ 17:01:01 DEBUG { 17:01:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:01:01 DEBUG }, 17:01:01 DEBUG "instance": "10.142.0.107:9100" 17:01:01 DEBUG "value": [ 17:01:01 DEBUG }, 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "pod": "ds-idrepo-0" 17:01:01 DEBUG }, 17:01:01 DEBUG "value": [ 17:01:01 DEBUG }, 17:01:01 DEBUG "metric": { 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "result": [ 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "pod": "ds-idrepo-0" 17:01:01 DEBUG 17:01:01 DEBUG "metric": { 17:01:01 DEBUG }, 17:01:01 DEBUG "value": [ 17:01:01 DEBUG }, 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 17:01:01 DEBUG }, 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "pod": "ds-cts-0" 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG { 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG }, 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG "metric": { 17:01:01 DEBUG 17:01:01 DEBUG }, 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG }, 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG [...] 17:01:01 DEBUG [...] 17:01:01 DEBUG [...] 17:01:01 DEBUG }, 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG }, 17:01:01 DEBUG [...] 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG "pod": "ds-cts-2" 17:01:01 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:01:01 DEBUG 17:01:01 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG }, 17:01:01 DEBUG "pod": "ds-idrepo-2" 17:01:01 DEBUG }, 17:01:01 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:01:01 DEBUG "instance": "10.142.0.43:9100" 17:01:01 DEBUG 17:01:01 DEBUG "instance": "10.142.0.43:9100" 17:01:01 DEBUG 17:01:01 DEBUG "instance": "10.142.0.43:9100" 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG 17:01:01 DEBUG }, 17:01:01 DEBUG }, 17:01:01 DEBUG "instance": "10.142.0.43:9100" 17:01:01 DEBUG }, 17:01:01 DEBUG "pod": "ds-idrepo-2" 17:01:01 DEBUG 17:01:01 DEBUG }, 17:01:01 DEBUG "pod": "ds-cts-2" 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:01:01 DEBUG "value": [ 17:01:01 DEBUG }, 17:01:01 DEBUG }, 17:01:01 DEBUG [...] 17:01:01 DEBUG }, 17:01:01 DEBUG 17:01:01 DEBUG }, 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG 17:01:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "value": [ 17:01:01 DEBUG }, 17:01:01 DEBUG "value": [ 17:01:01 DEBUG }, 17:01:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:01:01 DEBUG "value": [ 17:01:01 DEBUG }, 17:01:01 DEBUG 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG }, 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "939343872" 17:01:01 DEBUG 17:01:01 DEBUG }, 17:01:01 DEBUG "instance": "10.142.0.43:9100" 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "pod": "ds-cts-2" 17:01:01 DEBUG "2341.7" 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "0" 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG "pod": "ds-idrepo-2" 17:01:01 DEBUG }, 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:01:01 DEBUG "value": [ 17:01:01 DEBUG }, 17:01:01 DEBUG "0" 17:01:01 DEBUG "0" 17:01:01 DEBUG "1629299131392" 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG "0" 17:01:01 DEBUG }, 17:01:01 DEBUG "0" 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG }, 17:01:01 DEBUG ] 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG ] 17:01:01 DEBUG "0" 17:01:01 DEBUG "57148665.266666666" 17:01:01 DEBUG 17:01:01 DEBUG "0.038999999999998626" 17:01:01 DEBUG ] 17:01:01 DEBUG "value": [ 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG }, 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG "0.053566666666678954" 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG "0.00003566081393765952" 17:01:01 DEBUG ] 17:01:01 DEBUG "value": [ 17:01:01 DEBUG ] 17:01:01 DEBUG "0.00004198666666666666" 17:01:01 DEBUG "value": [ 17:01:01 DEBUG } 17:01:01 DEBUG "0" 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG "0" 17:01:01 DEBUG "value": [ 17:01:01 DEBUG "2341.7" 17:01:01 DEBUG "value": [ 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG }, 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG ] 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG "0" 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG "value": [ 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG "0.006" 17:01:01 DEBUG } 17:01:01 DEBUG "57154747.03333333" 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG "106797" 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG 1713542461, 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG "7.896387021043073" 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG ] 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:01:01 DEBUG } 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 ERROR [http_cmd]: ERROR 17:02:01 ERROR --- status code --- 17:02:01 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:02:01 ERROR --- http response --- 17:02:01 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:02:01 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 DEBUG --- status code --- 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "data": { 17:02:01 DEBUG "data": { 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "result": [ 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "pod": "ds-idrepo-0" 17:02:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:02:01 DEBUG }, 17:02:01 DEBUG }, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG "pod": "ds-cts-2" 17:02:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:02:01 DEBUG }, 17:02:01 DEBUG }, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG "0" 17:02:01 DEBUG "2324.8333333333335" 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 INFO 17:02:01 INFO [loop_until]: kubectl --namespace=xlou top pods 17:02:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:02:01 INFO 17:02:01 INFO [loop_until]: kubectl --namespace=xlou top node 17:02:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713542521" 17:02:01 INFO [loop_until]: OK (rc = 0) 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713542521" 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 DEBUG --- stdout --- 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG --- status code --- 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11332m 7233Mi am-6fd554f8f7-btd4s 11145m 6911Mi am-6fd554f8f7-wxcsj 12570m 7062Mi ds-cts-0 5960m 8752Mi ds-cts-1 6185m 8954Mi ds-cts-2 6356m 7785Mi ds-idrepo-0 5562m 3420Mi ds-idrepo-1 5499m 3294Mi ds-idrepo-2 5593m 3364Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-79cd4858c7-2r8x4 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7853m 809Mi 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG --- stderr --- 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 INFO [http_cmd]: http status code OK 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG --- status code --- 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG { 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG { 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG --- http response --- 17:02:01 DEBUG { 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG { 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "data": { 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:02:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG { 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "data": { 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG { 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG ----- output ----- 17:02:01 DEBUG "data": { 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "data": { 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "data": { 17:02:01 DEBUG "data": { 17:02:01 DEBUG "data": { 17:02:01 DEBUG "data": { 17:02:01 DEBUG "data": { 17:02:01 DEBUG { 17:02:01 DEBUG "data": { 17:02:01 DEBUG "data": { 17:02:01 DEBUG "data": { 17:02:01 DEBUG "data": { 17:02:01 DEBUG "data": { 17:02:01 DEBUG "data": { 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "data": { 17:02:01 DEBUG "data": { 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "data": { 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "result": [ 17:02:01 DEBUG { 17:02:01 DEBUG "data": { 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "status": "success", 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "data": { 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "result": [ 17:02:01 DEBUG { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "data": { 17:02:01 DEBUG "data": { 17:02:01 DEBUG { 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "result": [ 17:02:01 DEBUG { 17:02:01 DEBUG "result": [ 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "pod": "ds-idrepo-0" 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "resultType": "vector", 17:02:01 DEBUG "metric": { 17:02:01 DEBUG { 17:02:01 DEBUG { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "pod": "ds-idrepo-0" 17:02:01 DEBUG }, 17:02:01 DEBUG { 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "result": [ 17:02:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "pod": "ds-idrepo-0" 17:02:01 DEBUG "metric": { 17:02:01 DEBUG "instance": "10.142.0.107:9100" 17:02:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:02:01 DEBUG }, 17:02:01 DEBUG "instance": "10.142.0.107:9100" 17:02:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:02:01 DEBUG { 17:02:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:02:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:02:01 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:02:01 DEBUG "pod": "ds-cts-0" 17:02:01 DEBUG "instance": "10.142.0.107:9100" 17:02:01 DEBUG "pod": "ds-cts-0" 17:02:01 DEBUG }, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "metric": { 17:02:01 DEBUG { 17:02:01 DEBUG "metric": { 17:02:01 DEBUG }, 17:02:01 DEBUG "instance": "10.142.0.107:9100" 17:02:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:02:01 DEBUG }, 17:02:01 DEBUG "pod": "ds-cts-0" 17:02:01 DEBUG }, 17:02:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:02:01 DEBUG "value": [ 17:02:01 DEBUG }, 17:02:01 DEBUG }, 17:02:01 DEBUG "metric": { 17:02:01 DEBUG }, 17:02:01 DEBUG }, 17:02:01 DEBUG }, 17:02:01 DEBUG }, 17:02:01 DEBUG }, 17:02:01 DEBUG }, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 17:02:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:02:01 DEBUG }, 17:02:01 DEBUG "instance": "10.142.0.107:9100" 17:02:01 DEBUG "value": [ 17:02:01 DEBUG }, 17:02:01 DEBUG }, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG }, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG }, 17:02:01 DEBUG 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG { 17:02:01 DEBUG "value": [ 17:02:01 DEBUG }, 17:02:01 DEBUG 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG }, 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG "metric": { 17:02:01 DEBUG 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG }, 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG [...] 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG "value": [ 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG "pod": "ds-idrepo-2" 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG }, 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 INFO [loop_until]: OK (rc = 0) 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:02:01 DEBUG 17:02:01 DEBUG "pod": "ds-cts-2" 17:02:01 DEBUG 17:02:01 DEBUG "pod": "ds-idrepo-2" 17:02:01 DEBUG }, 17:02:01 DEBUG 17:02:01 DEBUG [...] 17:02:01 DEBUG 17:02:01 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG "pod": "ds-idrepo-2" 17:02:01 DEBUG 17:02:01 DEBUG "instance": "10.142.0.43:9100" 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:02:01 DEBUG 17:02:01 DEBUG --- stdout --- 17:02:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:02:01 DEBUG 17:02:01 DEBUG }, 17:02:01 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:02:01 DEBUG }, 17:02:01 DEBUG "pod": "ds-cts-2" 17:02:01 DEBUG }, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG 17:02:01 DEBUG }, 17:02:01 DEBUG "instance": "10.142.0.43:9100" 17:02:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:02:01 DEBUG }, 17:02:01 DEBUG "pod": "ds-cts-2" 17:02:01 DEBUG }, 17:02:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:02:01 DEBUG "instance": "10.142.0.43:9100" 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG }, 17:02:01 DEBUG 17:02:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 60m 0% 6182Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5461m 18% 6978Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 48m 0% 2681Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11419m 38% 10496Mi 9% gke-xlou-cdm-default-pool-1b0769c8-w15q 6139m 20% 11795Mi 10% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5187Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11212m 37% 9767Mi 8% gke-xlou-cdm-ds-0f601751-cf7b 12294m 41% 2982Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13206m 44% 9264Mi 8% gke-xlou-cdm-ds-0f601751-j488 6235m 20% 6576Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 6151m 20% 13821Mi 12% gke-xlou-cdm-ds-0f601751-nj6q 6332m 21% 7106Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 6426m 21% 10280Mi 9% gke-xlou-cdm-frontend-a8771548-i82n 8026m 50% 4628Mi 7% 17:02:01 DEBUG }, 17:02:01 DEBUG 17:02:01 DEBUG "value": [ 17:02:01 DEBUG }, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG }, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:02:01 DEBUG 17:02:01 DEBUG "instance": "10.142.0.43:9100" 17:02:01 DEBUG "value": [ 17:02:01 DEBUG }, 17:02:01 DEBUG }, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG }, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG }, 17:02:01 DEBUG }, 17:02:01 DEBUG "944934912" 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:02:01 DEBUG --- stderr --- 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "instance": "10.142.0.43:9100" 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG "0" 17:02:01 DEBUG ] 17:02:01 DEBUG 17:02:01 DEBUG }, 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG ] 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG }, 17:02:01 DEBUG 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG }, 17:02:01 DEBUG "0" 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG "0" 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG "0" 17:02:01 DEBUG }, 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:02:01 DEBUG "0" 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG "0" 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG "1631831553024" 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "2324.8" 17:02:01 DEBUG } 17:02:01 DEBUG "0" 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "0" 17:02:01 DEBUG "value": [ 17:02:01 DEBUG ] 17:02:01 DEBUG "0" 17:02:01 DEBUG ] 17:02:01 DEBUG "0" 17:02:01 DEBUG ] 17:02:01 DEBUG "value": [ 17:02:01 DEBUG "value": [ 17:02:01 DEBUG } 17:02:01 DEBUG }, 17:02:01 DEBUG ] 17:02:01 DEBUG "0.03833333333333636" 17:02:01 DEBUG "246845.5" 17:02:01 DEBUG ] 17:02:01 DEBUG "0" 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG ] 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG } 17:02:01 DEBUG "value": [ 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG 1713542521, 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG "7.868787739092493" 17:02:01 DEBUG } 17:02:01 DEBUG "0.05369999999993808" 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG "2324.733333333333" 17:02:01 DEBUG "56818755.4" 17:02:01 DEBUG "0.006" 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG "56813613" 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG ] 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:02:01 DEBUG } 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 ERROR [http_cmd]: ERROR 17:03:01 ERROR --- status code --- 17:03:01 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:03:01 ERROR --- http response --- 17:03:01 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:03:01 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG { 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "data": { 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "result": [ 17:03:01 DEBUG { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "pod": "ds-idrepo-0" 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "pod": "ds-cts-2" 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG "0" 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG { 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "data": { 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "result": [ 17:03:01 DEBUG { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG "2341.2666666666664" 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 INFO 17:03:01 INFO [loop_until]: kubectl --namespace=xlou top pods 17:03:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 INFO [loop_until]: kubectl --namespace=xlou top node 17:03:01 DEBUG --- status code --- 17:03:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG { 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "data": { 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "result": [ 17:03:01 DEBUG { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG "0" 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542581" 17:03:01 INFO [loop_until]: OK (rc = 0) 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG --- status code --- 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG --- stdout --- 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG --- status code --- 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG --- status code --- 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG --- status code --- 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG --- status code --- 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11131m 7286Mi am-6fd554f8f7-btd4s 11386m 6943Mi am-6fd554f8f7-wxcsj 12096m 7116Mi ds-cts-0 6369m 8764Mi ds-cts-1 6734m 8960Mi ds-cts-2 6859m 7801Mi ds-idrepo-0 5584m 3421Mi ds-idrepo-1 5550m 3294Mi ds-idrepo-2 5468m 3361Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-79cd4858c7-2r8x4 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8015m 812Mi 17:03:01 DEBUG --- status code --- 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- status code --- 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- stderr --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG --- http response --- 17:03:01 INFO [http_cmd]: http status code OK 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:03:01 DEBUG --- status code --- 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG --- http response --- 17:03:01 INFO [loop_until]: OK (rc = 0) 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG { 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:03:01 DEBUG --- stdout --- 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG { 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG --- http response --- 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6189Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5443m 18% 6984Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 46m 0% 2681Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 10745m 35% 10562Mi 9% gke-xlou-cdm-default-pool-1b0769c8-w15q 6252m 20% 11817Mi 10% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5195Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11336m 37% 9819Mi 8% gke-xlou-cdm-ds-0f601751-cf7b 12363m 41% 2963Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12887m 43% 9318Mi 8% gke-xlou-cdm-ds-0f601751-j488 6302m 21% 6570Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 6494m 21% 13826Mi 12% gke-xlou-cdm-ds-0f601751-nj6q 6416m 21% 7106Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 6753m 22% 10325Mi 9% gke-xlou-cdm-frontend-a8771548-i82n 8055m 50% 4627Mi 7% 17:03:01 DEBUG { 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG { 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:03:01 DEBUG { 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "data": { 17:03:01 DEBUG { 17:03:01 DEBUG --- stderr --- 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG { 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG { 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "data": { 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "data": { 17:03:01 DEBUG "data": { 17:03:01 DEBUG "data": { 17:03:01 DEBUG ----- output ----- 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG { 17:03:01 DEBUG "data": { 17:03:01 DEBUG "data": { 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "data": { 17:03:01 DEBUG 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "data": { 17:03:01 DEBUG "data": { 17:03:01 DEBUG "data": { 17:03:01 DEBUG "data": { 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "data": { 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "data": { 17:03:01 DEBUG "data": { 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG { 17:03:01 DEBUG "data": { 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "data": { 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "data": { 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "data": { 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "status": "success", 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "data": { 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "result": [ 17:03:01 DEBUG { 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG "result": [ 17:03:01 DEBUG { 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "result": [ 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG "data": { 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "result": [ 17:03:01 DEBUG { 17:03:01 DEBUG "result": [ 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG "result": [ 17:03:01 DEBUG { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG { 17:03:01 DEBUG { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "resultType": "vector", 17:03:01 DEBUG { 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:03:01 DEBUG { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "pod": "ds-idrepo-0" 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "instance": "10.142.0.107:9100" 17:03:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:03:01 DEBUG "pod": "ds-idrepo-0" 17:03:01 DEBUG "result": [ 17:03:01 DEBUG "metric": { 17:03:01 DEBUG { 17:03:01 DEBUG "pod": "ds-idrepo-0" 17:03:01 DEBUG "pod": "ds-idrepo-0" 17:03:01 DEBUG }, 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "pod": "ds-cts-0" 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "instance": "10.142.0.107:9100" 17:03:01 DEBUG "pod": "ds-cts-0" 17:03:01 DEBUG "pod": "ds-idrepo-0" 17:03:01 DEBUG "instance": "10.142.0.107:9100" 17:03:01 DEBUG "metric": { 17:03:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:03:01 DEBUG }, 17:03:01 DEBUG "instance": "10.142.0.107:9100" 17:03:01 DEBUG "instance": "10.142.0.107:9100" 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG { 17:03:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:03:01 DEBUG "metric": { 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "pod": "ds-cts-0" 17:03:01 DEBUG }, 17:03:01 DEBUG "pod": "ds-idrepo-0" 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "metric": { 17:03:01 DEBUG }, 17:03:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "value": [ 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "pod": "ds-idrepo-0" 17:03:01 DEBUG "value": [ 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG }, 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG [...] 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG [...] 17:03:01 DEBUG [...] 17:03:01 DEBUG "instance": "10.142.0.43:9100" 17:03:01 DEBUG [...] 17:03:01 DEBUG [...] 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG }, 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG [...] 17:03:01 DEBUG 17:03:01 DEBUG }, 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "pod": "ds-idrepo-2" 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "instance": "10.142.0.43:9100" 17:03:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:03:01 DEBUG "pod": "ds-idrepo-2" 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:03:01 DEBUG 17:03:01 DEBUG }, 17:03:01 DEBUG 17:03:01 DEBUG "pod": "ds-cts-2" 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG }, 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG 17:03:01 DEBUG 17:03:01 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 17:03:01 DEBUG }, 17:03:01 DEBUG "pod": "ds-idrepo-2" 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "pod": "ds-cts-2" 17:03:01 DEBUG }, 17:03:01 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 17:03:01 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:03:01 DEBUG "pod": "ds-cts-2" 17:03:01 DEBUG "56991959.766666666" 17:03:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:03:01 DEBUG "instance": "10.142.0.43:9100" 17:03:01 DEBUG "instance": "10.142.0.43:9100" 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "2341.8333333333335" 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:03:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG }, 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG ] 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG ] 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG }, 17:03:01 DEBUG }, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "386923.5" 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "value": [ 17:03:01 DEBUG } 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG "0.05736666666673121" 17:03:01 DEBUG "0" 17:03:01 DEBUG } 17:03:01 DEBUG "1634385601536" 17:03:01 DEBUG "949661696" 17:03:01 DEBUG "0" 17:03:01 DEBUG "value": [ 17:03:01 DEBUG "value": [ 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG "2341.633333333333" 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG ] 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG "0" 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG ] 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG "instance": "10.142.0.43:9100" 17:03:01 DEBUG ] 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG "490.44677576787467" 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG ] 17:03:01 DEBUG "0" 17:03:01 DEBUG } 17:03:01 DEBUG "0" 17:03:01 DEBUG ] 17:03:01 DEBUG "0" 17:03:01 DEBUG "0" 17:03:01 DEBUG "0" 17:03:01 DEBUG } 17:03:01 DEBUG "7.876285737247595" 17:03:01 DEBUG }, 17:03:01 DEBUG } 17:03:01 DEBUG "56997189.166666664" 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG "0" 17:03:01 DEBUG ] 17:03:01 DEBUG "0.007" 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG "value": [ 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG 1713542581, 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG "0.04266666666666575" 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG ] 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:03:01 DEBUG } 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 ERROR [http_cmd]: ERROR 17:04:01 ERROR --- status code --- 17:04:01 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:04:01 ERROR --- http response --- 17:04:01 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:04:01 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- http response --- 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG { 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "data": { 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "result": [ 17:04:01 DEBUG { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "pod": "ds-idrepo-0" 17:04:01 DEBUG }, 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:04:01 DEBUG "value": [ 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG 17:04:01 DEBUG { 17:04:01 DEBUG 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG 17:04:01 DEBUG "data": { 17:04:01 DEBUG [...] 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG 17:04:01 DEBUG "result": [ 17:04:01 DEBUG 17:04:01 DEBUG { 17:04:01 DEBUG 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "pod": "ds-cts-2" 17:04:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:04:01 DEBUG }, 17:04:01 DEBUG }, 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG 17:04:01 DEBUG "0" 17:04:01 DEBUG 17:04:01 DEBUG ] 17:04:01 DEBUG 17:04:01 DEBUG } 17:04:01 DEBUG [...] 17:04:01 DEBUG ] 17:04:01 DEBUG 17:04:01 DEBUG } 17:04:01 DEBUG 17:04:01 DEBUG } 17:04:01 DEBUG 17:04:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:04:01 DEBUG }, 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG "2338.233333333333" 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG { 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "data": { 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "result": [ 17:04:01 DEBUG { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:04:01 DEBUG }, 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:04:01 DEBUG }, 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG "0" 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 INFO 17:04:01 INFO [loop_until]: kubectl --namespace=xlou top pods 17:04:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:04:01 INFO 17:04:01 INFO [loop_until]: kubectl --namespace=xlou top node 17:04:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542641" 17:04:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713542641" 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [loop_until]: OK (rc = 0) 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 DEBUG --- status code --- 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [loop_until]: OK (rc = 0) 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG --- status code --- 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 DEBUG --- stdout --- 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG --- status code --- 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG --- stdout --- 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG --- status code --- 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11255m 7287Mi am-6fd554f8f7-btd4s 11317m 6945Mi am-6fd554f8f7-wxcsj 11883m 7128Mi ds-cts-0 6679m 9349Mi ds-cts-1 6686m 9348Mi ds-cts-2 6901m 9476Mi ds-idrepo-0 5629m 3421Mi ds-idrepo-1 5478m 3293Mi ds-idrepo-2 5532m 3398Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-79cd4858c7-2r8x4 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7909m 815Mi 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6178Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5488m 18% 6985Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 45m 0% 2681Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11108m 37% 10567Mi 9% gke-xlou-cdm-default-pool-1b0769c8-w15q 6660m 22% 12588Mi 11% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5197Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11420m 38% 9810Mi 8% gke-xlou-cdm-ds-0f601751-cf7b 12318m 41% 2960Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12885m 43% 9346Mi 8% gke-xlou-cdm-ds-0f601751-j488 6269m 20% 6565Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 6687m 22% 14385Mi 12% gke-xlou-cdm-ds-0f601751-nj6q 6396m 21% 7102Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 6880m 23% 11997Mi 10% gke-xlou-cdm-frontend-a8771548-i82n 8059m 50% 4631Mi 7% 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG --- stderr --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG --- stderr --- 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:04:01 INFO [http_cmd]: http status code OK 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:04:01 DEBUG 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG --- status code --- 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:04:01 DEBUG 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG { 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG { 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "data": { 17:04:01 DEBUG { 17:04:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG { 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG { 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG --- http response --- 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "data": { 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "data": { 17:04:01 DEBUG { 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG { 17:04:01 DEBUG "result": [ 17:04:01 DEBUG { 17:04:01 DEBUG "data": { 17:04:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "data": { 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "data": { 17:04:01 DEBUG "data": { 17:04:01 DEBUG "data": { 17:04:01 DEBUG "data": { 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "data": { 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG { 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "data": { 17:04:01 DEBUG ----- output ----- 17:04:01 DEBUG "data": { 17:04:01 DEBUG "data": { 17:04:01 DEBUG "data": { 17:04:01 DEBUG "data": { 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "data": { 17:04:01 DEBUG "data": { 17:04:01 DEBUG "data": { 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "data": { 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "data": { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "instance": "10.142.0.107:9100" 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG { 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "result": [ 17:04:01 DEBUG { 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG { 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG }, 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "status": "success", 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "pod": "ds-idrepo-0" 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "result": [ 17:04:01 DEBUG { 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "value": [ 17:04:01 DEBUG { 17:04:01 DEBUG "data": { 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "instance": "10.142.0.107:9100" 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "result": [ 17:04:01 DEBUG }, 17:04:01 DEBUG { 17:04:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:04:01 DEBUG { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "resultType": "vector", 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG { 17:04:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:04:01 DEBUG "instance": "10.142.0.107:9100" 17:04:01 DEBUG }, 17:04:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:04:01 DEBUG "pod": "ds-idrepo-0" 17:04:01 DEBUG "pod": "ds-idrepo-0" 17:04:01 DEBUG { 17:04:01 DEBUG { 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "metric": { 17:04:01 DEBUG }, 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG 17:04:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:04:01 DEBUG "result": [ 17:04:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:04:01 DEBUG "pod": "ds-cts-0" 17:04:01 DEBUG "metric": { 17:04:01 DEBUG }, 17:04:01 DEBUG }, 17:04:01 DEBUG "value": [ 17:04:01 DEBUG }, 17:04:01 DEBUG }, 17:04:01 DEBUG }, 17:04:01 DEBUG "metric": { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG 17:04:01 DEBUG "pod": "ds-cts-0" 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "pod": "ds-idrepo-0" 17:04:01 DEBUG }, 17:04:01 DEBUG "pod": "ds-cts-0" 17:04:01 DEBUG "instance": "10.142.0.107:9100" 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG { 17:04:01 DEBUG "metric": { 17:04:01 DEBUG }, 17:04:01 DEBUG "pod": "ds-idrepo-0" 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "pod": "ds-idrepo-0" 17:04:01 DEBUG "pod": "ds-idrepo-0" 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG "value": [ 17:04:01 DEBUG }, 17:04:01 DEBUG }, 17:04:01 DEBUG [...] 17:04:01 DEBUG "value": [ 17:04:01 DEBUG }, 17:04:01 DEBUG "instance": "10.142.0.107:9100" 17:04:01 DEBUG "value": [ 17:04:01 DEBUG }, 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG "value": [ 17:04:01 DEBUG }, 17:04:01 DEBUG 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG "value": [ 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG "instance": "10.142.0.43:9100" 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG "instance": "10.142.0.43:9100" 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG 17:04:01 DEBUG "pod": "ds-idrepo-2" 17:04:01 DEBUG [...] 17:04:01 DEBUG 17:04:01 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:04:01 DEBUG 17:04:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:04:01 DEBUG "instance": "10.142.0.43:9100" 17:04:01 DEBUG 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "pod": "ds-idrepo-2" 17:04:01 DEBUG }, 17:04:01 DEBUG 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG 17:04:01 DEBUG "pod": "ds-idrepo-2" 17:04:01 DEBUG 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG "instance": "10.142.0.43:9100" 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG 17:04:01 DEBUG "pod": "ds-cts-2" 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG }, 17:04:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG }, 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 17:04:01 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "pod": "ds-cts-2" 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:04:01 DEBUG }, 17:04:01 DEBUG "pod": "ds-cts-2" 17:04:01 DEBUG "56891680" 17:04:01 DEBUG }, 17:04:01 DEBUG 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "value": [ 17:04:01 DEBUG }, 17:04:01 DEBUG "1636962823168" 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG 17:04:01 DEBUG }, 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG }, 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG }, 17:04:01 DEBUG "value": [ 17:04:01 DEBUG }, 17:04:01 DEBUG ] 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG "instance": "10.142.0.43:9100" 17:04:01 DEBUG "value": [ 17:04:01 DEBUG }, 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG "value": [ 17:04:01 DEBUG ] 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG "0" 17:04:01 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "0" 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "2338.6" 17:04:01 DEBUG "value": [ 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG "value": [ 17:04:01 DEBUG } 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG "56897114.43333334" 17:04:01 DEBUG "2338.4333333333334" 17:04:01 DEBUG }, 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "953413632" 17:04:01 DEBUG "0.05616666666662221" 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG } 17:04:01 DEBUG "0" 17:04:01 DEBUG ] 17:04:01 DEBUG }, 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG ] 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG ] 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG "0" 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG ] 17:04:01 DEBUG }, 17:04:01 DEBUG ] 17:04:01 DEBUG ] 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "0" 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG ] 17:04:01 DEBUG ] 17:04:01 DEBUG "527130.5" 17:04:01 DEBUG ] 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG "value": [ 17:04:01 DEBUG "0.000014739342079625974" 17:04:01 DEBUG } 17:04:01 DEBUG "0.00002099333333333333" 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG "0" 17:04:01 DEBUG } 17:04:01 DEBUG "value": [ 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG ] 17:04:01 DEBUG "0" 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG ] 17:04:01 DEBUG "7.862591614750549" 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG ] 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG 1713542641, 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG ] 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG "0.007" 17:04:01 DEBUG } 17:04:01 DEBUG "0.04133333333333743" 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG ] 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG "0" 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG ] 17:04:01 DEBUG ] 17:04:01 DEBUG ] 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:04:01 DEBUG } 17:05:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542701" 17:05:01 INFO [http_cmd]: http status code OK 17:05:01 DEBUG --- status code --- 17:05:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:01 DEBUG --- http response --- 17:05:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:05:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713542701" 17:05:01 ERROR [http_cmd]: ERROR 17:05:01 ERROR --- status code --- 17:05:01 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:05:01 ERROR --- http response --- 17:05:01 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:05:01 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:05:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542701" 17:05:01 INFO [http_cmd]: http status code OK 17:05:01 DEBUG --- status code --- 17:05:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:01 DEBUG --- http response --- 17:05:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:05:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542701" 17:05:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713542701" 17:05:01 INFO [http_cmd]: http status code OK 17:05:01 INFO [http_cmd]: http status code OK 17:05:01 DEBUG --- status code --- 17:05:01 DEBUG --- status code --- 17:05:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:01 DEBUG --- http response --- 17:05:01 DEBUG --- http response --- 17:05:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:05:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:05:01 DEBUG ----- output ----- 17:05:01 DEBUG ----- output ----- 17:05:01 DEBUG { 17:05:01 DEBUG { 17:05:01 DEBUG "status": "success", 17:05:01 DEBUG "status": "success", 17:05:01 DEBUG "data": { 17:05:01 DEBUG "data": { 17:05:01 DEBUG "resultType": "vector", 17:05:01 DEBUG "resultType": "vector", 17:05:01 DEBUG "result": [ 17:05:01 DEBUG "result": [ 17:05:01 DEBUG { 17:05:01 DEBUG { 17:05:01 DEBUG "metric": { 17:05:01 DEBUG "metric": { 17:05:01 DEBUG "pod": "ds-idrepo-0" 17:05:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:05:01 DEBUG }, 17:05:01 DEBUG }, 17:05:01 DEBUG "value": [ 17:05:01 DEBUG "value": [ 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG [...] 17:05:01 DEBUG [...] 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG "pod": "ds-cts-2" 17:05:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:05:01 DEBUG }, 17:05:01 DEBUG }, 17:05:01 DEBUG "value": [ 17:05:01 DEBUG "value": [ 17:05:01 DEBUG 1713542701, 17:05:01 DEBUG 1713542701, 17:05:01 DEBUG "0" 17:05:01 DEBUG "2328.6666666666665" 17:05:01 DEBUG ] 17:05:01 DEBUG ] 17:05:01 DEBUG } 17:05:01 DEBUG } 17:05:01 DEBUG ] 17:05:01 DEBUG ] 17:05:01 DEBUG } 17:05:01 DEBUG } 17:05:01 DEBUG } 17:05:01 DEBUG } 17:05:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542701" 17:05:01 INFO [http_cmd]: http status code OK 17:05:01 DEBUG --- status code --- 17:05:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:01 DEBUG --- http response --- 17:05:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:05:01 DEBUG ----- output ----- 17:05:01 DEBUG { 17:05:01 DEBUG "status": "success", 17:05:01 DEBUG "data": { 17:05:01 DEBUG "resultType": "vector", 17:05:01 DEBUG "result": [ 17:05:01 DEBUG { 17:05:01 DEBUG "metric": { 17:05:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:05:01 DEBUG }, 17:05:01 DEBUG "value": [ 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG [...] 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG 17:05:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:05:01 DEBUG }, 17:05:01 DEBUG "value": [ 17:05:01 DEBUG 1713542701, 17:05:01 DEBUG "0" 17:05:01 DEBUG ] 17:05:01 DEBUG } 17:05:01 DEBUG ] 17:05:01 DEBUG } 17:05:01 DEBUG } 17:05:01 INFO 17:05:01 INFO [loop_until]: kubectl --namespace=xlou top node 17:05:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:05:01 INFO 17:05:01 INFO [loop_until]: kubectl --namespace=xlou top pods 17:05:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542702" 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG { 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG "data": { 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "result": [ 17:05:02 DEBUG { 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "pod": "ds-idrepo-1" 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG }, 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713542702" 17:05:02 DEBUG "value": [ 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:05:02 DEBUG "0" 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG ] 17:05:02 DEBUG { 17:05:02 DEBUG } 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG ] 17:05:02 DEBUG "data": { 17:05:02 DEBUG } 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG } 17:05:02 DEBUG "result": [ 17:05:02 DEBUG { 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "instance": "10.142.0.107:9100" 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "instance": "10.142.0.43:9100" 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG "0.04100000000000439" 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713542702" 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713542702" 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 DEBUG --- status code --- 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 DEBUG --- status code --- 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG --- status code --- 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG --- status code --- 17:05:02 INFO [loop_until]: OK (rc = 0) 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG --- status code --- 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG --- stdout --- 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11114m 8655Mi am-6fd554f8f7-btd4s 11119m 8712Mi am-6fd554f8f7-wxcsj 11644m 8726Mi ds-cts-0 6525m 11182Mi ds-cts-1 6728m 11181Mi ds-cts-2 7080m 11475Mi ds-idrepo-0 5592m 3421Mi ds-idrepo-1 5408m 3321Mi ds-idrepo-2 5349m 3377Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7885m 818Mi 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:05:02 DEBUG --- stderr --- 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:05:02 INFO [loop_until]: OK (rc = 0) 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG { 17:05:02 DEBUG { 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG --- stdout --- 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG { 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG { 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG { 17:05:02 DEBUG { 17:05:02 DEBUG { 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG { 17:05:02 DEBUG ----- output ----- 17:05:02 INFO [http_cmd]: http status code OK 17:05:02 DEBUG { 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG { 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG { 17:05:02 DEBUG { 17:05:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6165Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5443m 18% 6984Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 47m 0% 2686Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11343m 37% 11927Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 6676m 22% 14469Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5204Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11209m 37% 11580Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12365m 41% 2981Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12696m 42% 10919Mi 9% gke-xlou-cdm-ds-0f601751-j488 6225m 20% 6582Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 6774m 22% 16239Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 6369m 21% 7088Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 6949m 23% 14019Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8020m 50% 4628Mi 7% 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG { 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG { 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG { 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG { 17:05:02 DEBUG --- status code --- 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG { 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG "data": { 17:05:02 DEBUG "data": { 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG --- stderr --- 17:05:02 DEBUG "data": { 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:05:02 DEBUG "data": { 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG "data": { 17:05:02 DEBUG "data": { 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG "data": { 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:05:02 DEBUG "data": { 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG "data": { 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG "result": [ 17:05:02 DEBUG "data": { 17:05:02 DEBUG 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "data": { 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "data": { 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "result": [ 17:05:02 DEBUG "data": { 17:05:02 DEBUG "data": { 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "data": { 17:05:02 DEBUG --- http response --- 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "data": { 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "result": [ 17:05:02 DEBUG "data": { 17:05:02 DEBUG { 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "result": [ 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG { 17:05:02 DEBUG "result": [ 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "result": [ 17:05:02 DEBUG { 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "result": [ 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "result": [ 17:05:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "result": [ 17:05:02 DEBUG { 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "result": [ 17:05:02 DEBUG { 17:05:02 DEBUG "result": [ 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG { 17:05:02 DEBUG "result": [ 17:05:02 DEBUG { 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "result": [ 17:05:02 DEBUG "result": [ 17:05:02 DEBUG { 17:05:02 DEBUG "result": [ 17:05:02 DEBUG { 17:05:02 DEBUG ----- output ----- 17:05:02 DEBUG "result": [ 17:05:02 DEBUG { 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "result": [ 17:05:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:05:02 DEBUG }, 17:05:02 DEBUG "metric": { 17:05:02 DEBUG { 17:05:02 DEBUG "data": { 17:05:02 DEBUG "metric": { 17:05:02 DEBUG { 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "instance": "10.142.0.107:9100" 17:05:02 DEBUG { 17:05:02 DEBUG { 17:05:02 DEBUG "metric": { 17:05:02 DEBUG { 17:05:02 DEBUG "metric": { 17:05:02 DEBUG { 17:05:02 DEBUG { 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "pod": "ds-idrepo-0" 17:05:02 DEBUG { 17:05:02 DEBUG "value": [ 17:05:02 DEBUG "pod": "ds-cts-0" 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG { 17:05:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "pod": "ds-idrepo-0" 17:05:02 DEBUG }, 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "pod": "ds-idrepo-1" 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "instance": "10.142.0.107:9100" 17:05:02 DEBUG "status": "success", 17:05:02 DEBUG "metric": { 17:05:02 DEBUG "pod": "ds-idrepo-0" 17:05:02 DEBUG }, 17:05:02 DEBUG "metric": { 17:05:02 DEBUG 17:05:02 DEBUG }, 17:05:02 DEBUG "pod": "ds-cts-0" 17:05:02 DEBUG "result": [ 17:05:02 DEBUG "metric": { 17:05:02 DEBUG }, 17:05:02 DEBUG "pod": "ds-idrepo-1" 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:05:02 DEBUG "instance": "10.142.0.107:9100" 17:05:02 DEBUG }, 17:05:02 DEBUG "pod": "ds-cts-0" 17:05:02 DEBUG }, 17:05:02 DEBUG "data": { 17:05:02 DEBUG "pod": "ds-idrepo-1" 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG }, 17:05:02 DEBUG { 17:05:02 DEBUG "instance": "10.142.0.107:9100" 17:05:02 DEBUG "value": [ 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG }, 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG "resultType": "vector", 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG }, 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG "metric": { 17:05:02 DEBUG }, 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG "value": [ 17:05:02 DEBUG }, 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "result": [ 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG { 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG }, 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG "metric": { 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG }, 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG }, 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG [...] 17:05:02 DEBUG [...] 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "instance": "10.142.0.43:9100" 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "pod": "am-6fd554f8f7-btd4s" 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG "pod": "ds-cts-2" 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:05:02 DEBUG }, 17:05:02 DEBUG "pod": "ds-idrepo-2" 17:05:02 DEBUG }, 17:05:02 DEBUG 17:05:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:05:02 DEBUG 17:05:02 DEBUG }, 17:05:02 DEBUG "instance": "10.142.0.43:9100" 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "pod": "ds-idrepo-2" 17:05:02 DEBUG }, 17:05:02 DEBUG 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG }, 17:05:02 DEBUG "pod": "ds-cts-2" 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG }, 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG }, 17:05:02 DEBUG 17:05:02 DEBUG "pod": "ds-idrepo-2" 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:05:02 DEBUG "953778176" 17:05:02 DEBUG "value": [ 17:05:02 DEBUG }, 17:05:02 DEBUG 17:05:02 DEBUG 17:05:02 DEBUG "pod": "am-6fd554f8f7-btd4s" 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG "instance": "10.142.0.43:9100" 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG "pod": "ds-cts-2" 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG [...] 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG }, 17:05:02 DEBUG ] 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:05:02 DEBUG "instance": "10.142.0.43:9100" 17:05:02 DEBUG }, 17:05:02 DEBUG "2328.4333333333334" 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG "56660584.6" 17:05:02 DEBUG ] 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG "0" 17:05:02 DEBUG "value": [ 17:05:02 DEBUG } 17:05:02 DEBUG "0" 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG "0" 17:05:02 DEBUG } 17:05:02 DEBUG "2328.5333333333333" 17:05:02 DEBUG "value": [ 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG "559.4872836629423" 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG "0" 17:05:02 DEBUG ] 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG ] 17:05:02 DEBUG ] 17:05:02 DEBUG "0" 17:05:02 DEBUG }, 17:05:02 DEBUG "value": [ 17:05:02 DEBUG ] 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG ] 17:05:02 DEBUG ] 17:05:02 DEBUG ] 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG "1639558677504" 17:05:02 DEBUG ] 17:05:02 DEBUG "0" 17:05:02 DEBUG 17:05:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG "667194" 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG "7.914693758612622" 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG } 17:05:02 DEBUG "0" 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG "0.05693333333329633" 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG "0" 17:05:02 DEBUG }, 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG "56666189.4" 17:05:02 DEBUG ] 17:05:02 DEBUG ] 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG "value": [ 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG ] 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG 1713542702, 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG "0.007" 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG } 17:05:02 DEBUG ] 17:05:02 DEBUG } 17:05:02 DEBUG } 17:06:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542761" 17:06:01 INFO [http_cmd]: http status code OK 17:06:01 DEBUG --- status code --- 17:06:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:01 DEBUG --- http response --- 17:06:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:06:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713542761" 17:06:01 ERROR [http_cmd]: ERROR 17:06:01 ERROR --- status code --- 17:06:01 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:06:01 ERROR --- http response --- 17:06:01 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:06:01 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:06:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542761" 17:06:01 INFO [http_cmd]: http status code OK 17:06:01 DEBUG --- status code --- 17:06:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:01 DEBUG --- http response --- 17:06:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:06:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542761" 17:06:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713542761" 17:06:01 INFO [http_cmd]: http status code OK 17:06:01 DEBUG --- status code --- 17:06:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:01 DEBUG --- http response --- 17:06:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:06:01 DEBUG ----- output ----- 17:06:01 DEBUG { 17:06:01 DEBUG "status": "success", 17:06:01 DEBUG "data": { 17:06:01 DEBUG "resultType": "vector", 17:06:01 DEBUG "result": [ 17:06:01 DEBUG { 17:06:01 DEBUG "metric": { 17:06:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:06:01 DEBUG }, 17:06:01 DEBUG "value": [ 17:06:01 DEBUG 17:06:01 DEBUG 17:06:01 DEBUG 17:06:01 DEBUG [...] 17:06:01 DEBUG 17:06:01 DEBUG 17:06:01 DEBUG 17:06:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:06:01 DEBUG }, 17:06:01 DEBUG "value": [ 17:06:01 DEBUG 1713542761, 17:06:01 DEBUG "2339.1" 17:06:01 DEBUG ] 17:06:01 DEBUG } 17:06:01 DEBUG ] 17:06:01 DEBUG } 17:06:01 DEBUG } 17:06:01 INFO [http_cmd]: http status code OK 17:06:01 DEBUG --- status code --- 17:06:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:01 DEBUG --- http response --- 17:06:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:06:01 DEBUG ----- output ----- 17:06:01 DEBUG { 17:06:01 DEBUG "status": "success", 17:06:01 DEBUG "data": { 17:06:01 DEBUG "resultType": "vector", 17:06:01 DEBUG "result": [ 17:06:01 DEBUG { 17:06:01 DEBUG "metric": { 17:06:01 DEBUG "pod": "ds-idrepo-0" 17:06:01 DEBUG }, 17:06:01 DEBUG "value": [ 17:06:01 DEBUG 17:06:01 DEBUG 17:06:01 DEBUG 17:06:01 DEBUG [...] 17:06:01 DEBUG 17:06:01 DEBUG 17:06:01 DEBUG 17:06:01 DEBUG "pod": "ds-cts-2" 17:06:01 DEBUG }, 17:06:01 DEBUG "value": [ 17:06:01 DEBUG 1713542761, 17:06:01 DEBUG "0" 17:06:01 DEBUG ] 17:06:01 DEBUG } 17:06:01 DEBUG ] 17:06:01 DEBUG } 17:06:01 DEBUG } 17:06:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542761" 17:06:01 INFO [http_cmd]: http status code OK 17:06:01 DEBUG --- status code --- 17:06:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:01 DEBUG --- http response --- 17:06:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:06:01 DEBUG ----- output ----- 17:06:01 DEBUG { 17:06:01 DEBUG "status": "success", 17:06:01 DEBUG "data": { 17:06:01 DEBUG "resultType": "vector", 17:06:01 DEBUG "result": [ 17:06:01 DEBUG { 17:06:01 DEBUG "metric": { 17:06:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:06:01 DEBUG }, 17:06:01 DEBUG "value": [ 17:06:01 DEBUG 17:06:01 DEBUG 17:06:01 DEBUG 17:06:01 DEBUG [...] 17:06:01 DEBUG 17:06:01 DEBUG 17:06:01 DEBUG 17:06:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:06:01 DEBUG }, 17:06:01 DEBUG "value": [ 17:06:01 DEBUG 1713542761, 17:06:01 DEBUG "0" 17:06:01 DEBUG ] 17:06:01 DEBUG } 17:06:01 DEBUG ] 17:06:01 DEBUG } 17:06:01 DEBUG } 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG { 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "data": { 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "result": [ 17:06:02 DEBUG { 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "pod": "ds-idrepo-0" 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "0" 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG { 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "data": { 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "result": [ 17:06:02 DEBUG { 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "instance": "10.142.0.107:9100" 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG "instance": "10.142.0.43:9100" 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "0.043000000000006366" 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 INFO 17:06:02 INFO [loop_until]: kubectl --namespace=xlou top pods 17:06:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:06:02 INFO 17:06:02 INFO [loop_until]: kubectl --namespace=xlou top node 17:06:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542762" 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [loop_until]: OK (rc = 0) 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [loop_until]: OK (rc = 0) 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- status code --- 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- stdout --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- status code --- 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 DEBUG --- stdout --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 INFO [http_cmd]: http status code OK 17:06:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 10862m 8685Mi am-6fd554f8f7-btd4s 11393m 8745Mi am-6fd554f8f7-wxcsj 11541m 8758Mi ds-cts-0 6706m 12357Mi ds-cts-1 6942m 12292Mi ds-cts-2 6980m 12348Mi ds-idrepo-0 5630m 3421Mi ds-idrepo-1 5490m 3339Mi ds-idrepo-2 5431m 3377Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7730m 821Mi 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6182Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5452m 18% 6986Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 47m 0% 2692Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11012m 36% 11945Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 6611m 22% 15437Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5216Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11218m 37% 11612Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12280m 41% 2986Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12712m 42% 10963Mi 9% gke-xlou-cdm-ds-0f601751-j488 6185m 20% 6587Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 6748m 22% 17178Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6417m 21% 7138Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7005m 23% 14883Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8021m 50% 4630Mi 7% 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- stderr --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- stderr --- 17:06:02 DEBUG --- status code --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:06:02 DEBUG 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:06:02 DEBUG 17:06:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:06:02 DEBUG { 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG --- http response --- 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG { 17:06:02 DEBUG { 17:06:02 DEBUG { 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG { 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG { 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:06:02 DEBUG { 17:06:02 DEBUG { 17:06:02 DEBUG { 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG { 17:06:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:06:02 DEBUG { 17:06:02 DEBUG { 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG { 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG { 17:06:02 DEBUG { 17:06:02 DEBUG { 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG { 17:06:02 DEBUG "data": { 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG ----- output ----- 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "data": { 17:06:02 DEBUG "data": { 17:06:02 DEBUG "data": { 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "data": { 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "data": { 17:06:02 DEBUG { 17:06:02 DEBUG "data": { 17:06:02 DEBUG "data": { 17:06:02 DEBUG "data": { 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "data": { 17:06:02 DEBUG { 17:06:02 DEBUG "data": { 17:06:02 DEBUG "data": { 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "data": { 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "data": { 17:06:02 DEBUG "data": { 17:06:02 DEBUG "data": { 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "data": { 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "status": "success", 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "data": { 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG { 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "data": { 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "result": [ 17:06:02 DEBUG { 17:06:02 DEBUG { 17:06:02 DEBUG { 17:06:02 DEBUG "result": [ 17:06:02 DEBUG { 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "result": [ 17:06:02 DEBUG { 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG { 17:06:02 DEBUG "result": [ 17:06:02 DEBUG { 17:06:02 DEBUG { 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "metric": { 17:06:02 DEBUG { 17:06:02 DEBUG "resultType": "vector", 17:06:02 DEBUG { 17:06:02 DEBUG { 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "metric": { 17:06:02 DEBUG { 17:06:02 DEBUG "metric": { 17:06:02 DEBUG { 17:06:02 DEBUG { 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "metric": { 17:06:02 DEBUG { 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "metric": { 17:06:02 DEBUG { 17:06:02 DEBUG "instance": "10.142.0.107:9100" 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "result": [ 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "pod": "ds-idrepo-0" 17:06:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:06:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "pod": "ds-idrepo-0" 17:06:02 DEBUG { 17:06:02 DEBUG "pod": "ds-idrepo-0" 17:06:02 DEBUG "metric": { 17:06:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:06:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:06:02 DEBUG "metric": { 17:06:02 DEBUG }, 17:06:02 DEBUG "pod": "ds-cts-0" 17:06:02 DEBUG { 17:06:02 DEBUG "pod": "ds-idrepo-0" 17:06:02 DEBUG "instance": "10.142.0.107:9100" 17:06:02 DEBUG }, 17:06:02 DEBUG }, 17:06:02 DEBUG }, 17:06:02 DEBUG "pod": "ds-idrepo-0" 17:06:02 DEBUG }, 17:06:02 DEBUG "pod": "ds-cts-0" 17:06:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:06:02 DEBUG }, 17:06:02 DEBUG "metric": { 17:06:02 DEBUG }, 17:06:02 DEBUG "instance": "10.142.0.107:9100" 17:06:02 DEBUG }, 17:06:02 DEBUG }, 17:06:02 DEBUG "pod": "ds-cts-0" 17:06:02 DEBUG "value": [ 17:06:02 DEBUG }, 17:06:02 DEBUG "metric": { 17:06:02 DEBUG }, 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG "value": [ 17:06:02 DEBUG "value": [ 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG }, 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG "pod": "ds-idrepo-0" 17:06:02 DEBUG "value": [ 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG "value": [ 17:06:02 DEBUG }, 17:06:02 DEBUG 17:06:02 DEBUG "value": [ 17:06:02 DEBUG "instance": "10.142.0.107:9100" 17:06:02 DEBUG "value": [ 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG }, 17:06:02 DEBUG 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG }, 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG [...] 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG "instance": "10.142.0.43:9100" 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG [...] 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG }, 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG "instance": "10.142.0.43:9100" 17:06:02 DEBUG 17:06:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:06:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:06:02 DEBUG 17:06:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:06:02 DEBUG 17:06:02 DEBUG "pod": "ds-cts-2" 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG "0.008" 17:06:02 DEBUG 17:06:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:06:02 DEBUG 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG 17:06:02 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:06:02 DEBUG }, 17:06:02 DEBUG "pod": "ds-idrepo-2" 17:06:02 DEBUG }, 17:06:02 DEBUG }, 17:06:02 DEBUG "pod": "ds-idrepo-2" 17:06:02 DEBUG }, 17:06:02 DEBUG 17:06:02 DEBUG }, 17:06:02 DEBUG 17:06:02 DEBUG "pod": "ds-idrepo-2" 17:06:02 DEBUG 17:06:02 DEBUG ] 17:06:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:06:02 DEBUG }, 17:06:02 DEBUG "pod": "ds-cts-2" 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "pod": "ds-cts-2" 17:06:02 DEBUG 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG "value": [ 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 17:06:02 DEBUG }, 17:06:02 DEBUG "instance": "10.142.0.43:9100" 17:06:02 DEBUG } 17:06:02 DEBUG }, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG }, 17:06:02 DEBUG "1642197011456" 17:06:02 DEBUG }, 17:06:02 DEBUG "instance": "10.142.0.43:9100" 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG }, 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:06:02 DEBUG "value": [ 17:06:02 DEBUG }, 17:06:02 DEBUG ] 17:06:02 DEBUG "value": [ 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG ] 17:06:02 DEBUG "value": [ 17:06:02 DEBUG }, 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "56902849.7" 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "807354" 17:06:02 DEBUG "955662336" 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "2339.366666666667" 17:06:02 DEBUG "value": [ 17:06:02 DEBUG "0" 17:06:02 DEBUG }, 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG } 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "2339.4666666666667" 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG } 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "value": [ 17:06:02 DEBUG "0" 17:06:02 DEBUG ] 17:06:02 DEBUG "0" 17:06:02 DEBUG ] 17:06:02 DEBUG ] 17:06:02 DEBUG "0" 17:06:02 DEBUG ] 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG ] 17:06:02 DEBUG "value": [ 17:06:02 DEBUG "0" 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG } 17:06:02 DEBUG "7.91416774585941" 17:06:02 DEBUG ] 17:06:02 DEBUG "0" 17:06:02 DEBUG ] 17:06:02 DEBUG "0" 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG "0.057433333333412405" 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG "0.000020177546049207553" 17:06:02 DEBUG } 17:06:02 DEBUG 1713542762, 17:06:02 DEBUG ] 17:06:02 DEBUG "56908374.4" 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG ] 17:06:02 DEBUG ] 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG ] 17:06:02 DEBUG ] 17:06:02 DEBUG "0" 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG ] 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG ] 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:06:02 DEBUG } 17:07:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542821" 17:07:01 INFO [http_cmd]: http status code OK 17:07:01 DEBUG --- status code --- 17:07:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:01 DEBUG --- http response --- 17:07:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:07:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713542821" 17:07:01 ERROR [http_cmd]: ERROR 17:07:01 ERROR --- status code --- 17:07:01 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:07:01 ERROR --- http response --- 17:07:01 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:07:01 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:07:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542821" 17:07:01 INFO [http_cmd]: http status code OK 17:07:01 DEBUG --- status code --- 17:07:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:01 DEBUG --- http response --- 17:07:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:07:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542821" 17:07:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713542821" 17:07:01 INFO [http_cmd]: http status code OK 17:07:01 INFO [http_cmd]: http status code OK 17:07:01 DEBUG --- status code --- 17:07:01 DEBUG --- status code --- 17:07:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:01 DEBUG --- http response --- 17:07:01 DEBUG --- http response --- 17:07:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:07:01 DEBUG ----- output ----- 17:07:01 DEBUG { 17:07:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:07:01 DEBUG "status": "success", 17:07:01 DEBUG ----- output ----- 17:07:01 DEBUG "data": { 17:07:01 DEBUG "resultType": "vector", 17:07:01 DEBUG { 17:07:01 DEBUG "result": [ 17:07:01 DEBUG "status": "success", 17:07:01 DEBUG { 17:07:01 DEBUG "data": { 17:07:01 DEBUG "metric": { 17:07:01 DEBUG "resultType": "vector", 17:07:01 DEBUG "pod": "ds-idrepo-0" 17:07:01 DEBUG "result": [ 17:07:01 DEBUG }, 17:07:01 DEBUG { 17:07:01 DEBUG "value": [ 17:07:01 DEBUG "metric": { 17:07:01 DEBUG 17:07:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:07:01 DEBUG 17:07:01 DEBUG }, 17:07:01 DEBUG 17:07:01 DEBUG "value": [ 17:07:01 DEBUG [...] 17:07:01 DEBUG 17:07:01 DEBUG 17:07:01 DEBUG 17:07:01 DEBUG 17:07:01 DEBUG 17:07:01 DEBUG 17:07:01 DEBUG [...] 17:07:01 DEBUG "pod": "ds-cts-2" 17:07:01 DEBUG 17:07:01 DEBUG }, 17:07:01 DEBUG 17:07:01 DEBUG "value": [ 17:07:01 DEBUG 17:07:01 DEBUG 1713542821, 17:07:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:07:01 DEBUG "0" 17:07:01 DEBUG }, 17:07:01 DEBUG ] 17:07:01 DEBUG "value": [ 17:07:01 DEBUG 1713542821, 17:07:01 DEBUG "2330.0333333333333" 17:07:01 DEBUG ] 17:07:01 DEBUG } 17:07:01 DEBUG } 17:07:01 DEBUG ] 17:07:01 DEBUG ] 17:07:01 DEBUG } 17:07:01 DEBUG } 17:07:01 DEBUG } 17:07:01 DEBUG } 17:07:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542821" 17:07:01 INFO [http_cmd]: http status code OK 17:07:01 DEBUG --- status code --- 17:07:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:01 DEBUG --- http response --- 17:07:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:07:01 DEBUG ----- output ----- 17:07:01 DEBUG { 17:07:01 DEBUG "status": "success", 17:07:01 DEBUG "data": { 17:07:01 DEBUG "resultType": "vector", 17:07:01 DEBUG "result": [ 17:07:01 DEBUG { 17:07:01 DEBUG "metric": { 17:07:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:07:01 DEBUG }, 17:07:01 DEBUG "value": [ 17:07:01 DEBUG 17:07:01 DEBUG 17:07:01 DEBUG 17:07:01 DEBUG [...] 17:07:01 DEBUG 17:07:01 DEBUG 17:07:01 DEBUG 17:07:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:07:01 DEBUG }, 17:07:01 DEBUG "value": [ 17:07:01 DEBUG 1713542821, 17:07:01 DEBUG "0" 17:07:01 DEBUG ] 17:07:01 DEBUG } 17:07:01 DEBUG ] 17:07:01 DEBUG } 17:07:01 DEBUG } 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG { 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "data": { 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "result": [ 17:07:02 DEBUG { 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:07:02 DEBUG }, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG "pod": "idm-b85f9d978-txpwv" 17:07:02 DEBUG }, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "0" 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG { 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "data": { 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "result": [ 17:07:02 DEBUG { 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "instance": "10.142.0.107:9100" 17:07:02 DEBUG }, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG "instance": "10.142.0.43:9100" 17:07:02 DEBUG }, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "0.039000000000004316" 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 INFO 17:07:02 INFO [loop_until]: kubectl --namespace=xlou top pods 17:07:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:07:02 INFO 17:07:02 INFO [loop_until]: kubectl --namespace=xlou top node 17:07:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542822" 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [loop_until]: OK (rc = 0) 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [loop_until]: OK (rc = 0) 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- stdout --- 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- stdout --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- status code --- 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11098m 8687Mi am-6fd554f8f7-btd4s 11148m 8747Mi am-6fd554f8f7-wxcsj 11544m 8760Mi ds-cts-0 6989m 12364Mi ds-cts-1 6939m 12297Mi ds-cts-2 7242m 12355Mi ds-idrepo-0 5573m 3421Mi ds-idrepo-1 5581m 3340Mi ds-idrepo-2 5279m 3382Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7617m 824Mi 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 INFO [http_cmd]: http status code OK 17:07:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6172Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5461m 18% 6979Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 49m 0% 2698Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 10942m 36% 11963Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 6915m 23% 15450Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 134m 0% 5186Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11355m 38% 11618Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12372m 41% 2968Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12295m 41% 10970Mi 9% gke-xlou-cdm-ds-0f601751-j488 6149m 20% 6597Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 6796m 22% 17171Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6373m 21% 7150Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7262m 24% 14880Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8036m 50% 4631Mi 7% 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- stderr --- 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- status code --- 17:07:02 DEBUG --- stderr --- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:07:02 DEBUG { 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:07:02 DEBUG 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:07:02 DEBUG 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG --- http response --- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:07:02 DEBUG { 17:07:02 DEBUG "data": { 17:07:02 DEBUG { 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:07:02 DEBUG { 17:07:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG { 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG ----- output ----- 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG "data": { 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "data": { 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "data": { 17:07:02 DEBUG "data": { 17:07:02 DEBUG "data": { 17:07:02 DEBUG "data": { 17:07:02 DEBUG "data": { 17:07:02 DEBUG "data": { 17:07:02 DEBUG { 17:07:02 DEBUG "data": { 17:07:02 DEBUG { 17:07:02 DEBUG "data": { 17:07:02 DEBUG "data": { 17:07:02 DEBUG "data": { 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG { 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "data": { 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "status": "success", 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "data": { 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "data": { 17:07:02 DEBUG "data": { 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "data": { 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "data": { 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG { 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG { 17:07:02 DEBUG "instance": "10.142.0.107:9100" 17:07:02 DEBUG { 17:07:02 DEBUG "result": [ 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG { 17:07:02 DEBUG "resultType": "vector", 17:07:02 DEBUG { 17:07:02 DEBUG { 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "metric": { 17:07:02 DEBUG }, 17:07:02 DEBUG "metric": { 17:07:02 DEBUG { 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "result": [ 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "result": [ 17:07:02 DEBUG { 17:07:02 DEBUG "metric": { 17:07:02 DEBUG { 17:07:02 DEBUG "pod": "ds-idrepo-0" 17:07:02 DEBUG { 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "instance": "10.142.0.107:9100" 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:07:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:07:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:07:02 DEBUG "pod": "ds-cts-0" 17:07:02 DEBUG "pod": "ds-idrepo-0" 17:07:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:07:02 DEBUG { 17:07:02 DEBUG "pod": "ds-cts-0" 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "pod": "ds-idrepo-0" 17:07:02 DEBUG "metric": { 17:07:02 DEBUG }, 17:07:02 DEBUG { 17:07:02 DEBUG "instance": "10.142.0.107:9100" 17:07:02 DEBUG }, 17:07:02 DEBUG 17:07:02 DEBUG }, 17:07:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:07:02 DEBUG }, 17:07:02 DEBUG }, 17:07:02 DEBUG }, 17:07:02 DEBUG }, 17:07:02 DEBUG }, 17:07:02 DEBUG }, 17:07:02 DEBUG "metric": { 17:07:02 DEBUG }, 17:07:02 DEBUG "pod": "ds-cts-0" 17:07:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:07:02 DEBUG }, 17:07:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "metric": { 17:07:02 DEBUG "instance": "10.142.0.107:9100" 17:07:02 DEBUG }, 17:07:02 DEBUG 17:07:02 DEBUG "value": [ 17:07:02 DEBUG }, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:07:02 DEBUG "value": [ 17:07:02 DEBUG }, 17:07:02 DEBUG }, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG }, 17:07:02 DEBUG 17:07:02 DEBUG "value": [ 17:07:02 DEBUG }, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG }, 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG "value": [ 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG [...] 17:07:02 DEBUG [...] 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG "instance": "10.142.0.43:9100" 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG }, 17:07:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:07:02 DEBUG 17:07:02 DEBUG [...] 17:07:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG "pod": "ds-idrepo-2" 17:07:02 DEBUG 17:07:02 DEBUG "instance": "10.142.0.43:9100" 17:07:02 DEBUG 17:07:02 DEBUG "instance": "10.142.0.43:9100" 17:07:02 DEBUG "value": [ 17:07:02 DEBUG }, 17:07:02 DEBUG 17:07:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:07:02 DEBUG 17:07:02 DEBUG }, 17:07:02 DEBUG "pod": "ds-idrepo-2" 17:07:02 DEBUG 17:07:02 DEBUG }, 17:07:02 DEBUG "pod": "ds-cts-2" 17:07:02 DEBUG 17:07:02 DEBUG 17:07:02 DEBUG "pod": "ds-idrepo-2" 17:07:02 DEBUG 17:07:02 DEBUG }, 17:07:02 DEBUG 17:07:02 DEBUG }, 17:07:02 DEBUG 17:07:02 DEBUG }, 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "1644830493696" 17:07:02 DEBUG "value": [ 17:07:02 DEBUG }, 17:07:02 DEBUG "pod": "ds-cts-2" 17:07:02 DEBUG "value": [ 17:07:02 DEBUG }, 17:07:02 DEBUG 17:07:02 DEBUG "value": [ 17:07:02 DEBUG }, 17:07:02 DEBUG "pod": "ds-cts-2" 17:07:02 DEBUG "pod": "ds-idrepo-1" 17:07:02 DEBUG }, 17:07:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "pod": "idm-b85f9d978-txpwv" 17:07:02 DEBUG ] 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG }, 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG }, 17:07:02 DEBUG }, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG }, 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "instance": "10.142.0.43:9100" 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG }, 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG }, 17:07:02 DEBUG } 17:07:02 DEBUG "947354.5" 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "960851968" 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "pod": "ds-idrepo-1" 17:07:02 DEBUG "2329.7" 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "0" 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "0" 17:07:02 DEBUG }, 17:07:02 DEBUG "56685842.43333334" 17:07:02 DEBUG ] 17:07:02 DEBUG "0.054899999999959925" 17:07:02 DEBUG "value": [ 17:07:02 DEBUG ] 17:07:02 DEBUG ] 17:07:02 DEBUG "2329.8333333333335" 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG ] 17:07:02 DEBUG "0" 17:07:02 DEBUG }, 17:07:02 DEBUG ] 17:07:02 DEBUG "0" 17:07:02 DEBUG "value": [ 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG ] 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG ] 17:07:02 DEBUG "value": [ 17:07:02 DEBUG "value": [ 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG "0" 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG "value": [ 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "0" 17:07:02 DEBUG } 17:07:02 DEBUG "0.008" 17:07:02 DEBUG } 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "56692423.46666667" 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG "0" 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG "6362.100766915834" 17:07:02 DEBUG ] 17:07:02 DEBUG ] 17:07:02 DEBUG ] 17:07:02 DEBUG ] 17:07:02 DEBUG 1713542822, 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG "0" 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG "7.8792023203205765" 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG ] 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG ] 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:07:02 DEBUG } 17:08:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542881" 17:08:01 INFO [http_cmd]: http status code OK 17:08:01 DEBUG --- status code --- 17:08:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:01 DEBUG --- http response --- 17:08:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:08:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713542881" 17:08:01 ERROR [http_cmd]: ERROR 17:08:01 ERROR --- status code --- 17:08:01 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:08:01 ERROR --- http response --- 17:08:01 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:08:01 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:08:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542881" 17:08:01 INFO [http_cmd]: http status code OK 17:08:01 DEBUG --- status code --- 17:08:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:01 DEBUG --- http response --- 17:08:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:08:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542881" 17:08:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713542881" 17:08:01 INFO [http_cmd]: http status code OK 17:08:01 DEBUG --- status code --- 17:08:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:01 DEBUG --- http response --- 17:08:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:08:01 DEBUG ----- output ----- 17:08:01 DEBUG { 17:08:01 DEBUG "status": "success", 17:08:01 DEBUG "data": { 17:08:01 DEBUG "resultType": "vector", 17:08:01 DEBUG "result": [ 17:08:01 DEBUG { 17:08:01 DEBUG "metric": { 17:08:01 DEBUG "pod": "ds-idrepo-0" 17:08:01 DEBUG }, 17:08:01 DEBUG "value": [ 17:08:01 DEBUG 17:08:01 DEBUG 17:08:01 DEBUG 17:08:01 DEBUG [...] 17:08:01 DEBUG 17:08:01 DEBUG 17:08:01 DEBUG 17:08:01 DEBUG "pod": "ds-cts-2" 17:08:01 DEBUG }, 17:08:01 DEBUG "value": [ 17:08:01 DEBUG 1713542881, 17:08:01 DEBUG "0" 17:08:01 DEBUG ] 17:08:01 DEBUG } 17:08:01 DEBUG ] 17:08:01 DEBUG } 17:08:01 DEBUG } 17:08:01 INFO [http_cmd]: http status code OK 17:08:01 DEBUG --- status code --- 17:08:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:01 DEBUG --- http response --- 17:08:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:08:01 DEBUG ----- output ----- 17:08:01 DEBUG { 17:08:01 DEBUG "status": "success", 17:08:01 DEBUG "data": { 17:08:01 DEBUG "resultType": "vector", 17:08:01 DEBUG "result": [ 17:08:01 DEBUG { 17:08:01 DEBUG "metric": { 17:08:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:08:01 DEBUG }, 17:08:01 DEBUG "value": [ 17:08:01 DEBUG 17:08:01 DEBUG 17:08:01 DEBUG 17:08:01 DEBUG [...] 17:08:01 DEBUG 17:08:01 DEBUG 17:08:01 DEBUG 17:08:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:08:01 DEBUG }, 17:08:01 DEBUG "value": [ 17:08:01 DEBUG 1713542881, 17:08:01 DEBUG "2337.4333333333334" 17:08:01 DEBUG ] 17:08:01 DEBUG } 17:08:01 DEBUG ] 17:08:01 DEBUG } 17:08:01 DEBUG } 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "data": { 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "result": [ 17:08:02 DEBUG { 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:08:02 DEBUG }, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:08:02 DEBUG }, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG "0" 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542882" 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "data": { 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "result": [ 17:08:02 DEBUG { 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "instance": "10.142.0.107:9100" 17:08:02 DEBUG }, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "instance": "10.142.0.43:9100" 17:08:02 DEBUG }, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG "0.03999999999999014" 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "data": { 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "result": [ 17:08:02 DEBUG { 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "pod": "ds-idrepo-1" 17:08:02 DEBUG }, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:08:02 DEBUG }, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG "0" 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 INFO 17:08:02 INFO [loop_until]: kubectl --namespace=xlou top pods 17:08:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:08:02 INFO 17:08:02 INFO [loop_until]: kubectl --namespace=xlou top node 17:08:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542882" 17:08:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542882" 17:08:02 INFO [loop_until]: OK (rc = 0) 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 INFO [loop_until]: OK (rc = 0) 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 DEBUG --- stdout --- 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- status code --- 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 DEBUG --- stdout --- 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11090m 8688Mi am-6fd554f8f7-btd4s 11038m 8753Mi am-6fd554f8f7-wxcsj 12053m 8763Mi ds-cts-0 7626m 12374Mi ds-cts-1 7567m 12307Mi ds-cts-2 8013m 12365Mi ds-idrepo-0 5632m 3421Mi ds-idrepo-1 5570m 3354Mi ds-idrepo-2 5329m 3385Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7768m 825Mi 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6174Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5495m 18% 6980Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 50m 0% 2685Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11027m 36% 11969Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 7479m 25% 15446Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5184Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11433m 38% 11616Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12376m 41% 2969Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12924m 43% 10971Mi 9% gke-xlou-cdm-ds-0f601751-j488 6228m 20% 6594Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 7714m 25% 17177Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6380m 21% 7161Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 7811m 26% 14885Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7793m 49% 4634Mi 7% 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG --- stderr --- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:08:02 INFO [http_cmd]: http status code OK 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG --- stderr --- 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG --- status code --- 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:08:02 DEBUG 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG { 17:08:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:08:02 DEBUG { 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:08:02 DEBUG { 17:08:02 DEBUG { 17:08:02 DEBUG { 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG --- http response --- 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG { 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG { 17:08:02 DEBUG { 17:08:02 DEBUG { 17:08:02 DEBUG { 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG { 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "data": { 17:08:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:08:02 DEBUG { 17:08:02 DEBUG { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG { 17:08:02 DEBUG "data": { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG { 17:08:02 DEBUG "data": { 17:08:02 DEBUG "data": { 17:08:02 DEBUG "data": { 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "data": { 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG ----- output ----- 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "data": { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "data": { 17:08:02 DEBUG "data": { 17:08:02 DEBUG "data": { 17:08:02 DEBUG "data": { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "data": { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "result": [ 17:08:02 DEBUG { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "result": [ 17:08:02 DEBUG { 17:08:02 DEBUG "data": { 17:08:02 DEBUG "data": { 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "data": { 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "data": { 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "data": { 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "data": { 17:08:02 DEBUG { 17:08:02 DEBUG "status": "success", 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG { 17:08:02 DEBUG { 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "instance": "10.142.0.107:9100" 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "data": { 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "result": [ 17:08:02 DEBUG { 17:08:02 DEBUG "result": [ 17:08:02 DEBUG { 17:08:02 DEBUG { 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "pod": "ds-idrepo-1" 17:08:02 DEBUG }, 17:08:02 DEBUG { 17:08:02 DEBUG { 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "metric": { 17:08:02 DEBUG { 17:08:02 DEBUG }, 17:08:02 DEBUG "result": [ 17:08:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:08:02 DEBUG "resultType": "vector", 17:08:02 DEBUG { 17:08:02 DEBUG { 17:08:02 DEBUG "metric": { 17:08:02 DEBUG { 17:08:02 DEBUG "metric": { 17:08:02 DEBUG { 17:08:02 DEBUG { 17:08:02 DEBUG "value": [ 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "metric": { 17:08:02 DEBUG { 17:08:02 DEBUG "instance": "10.142.0.107:9100" 17:08:02 DEBUG "pod": "ds-cts-0" 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "value": [ 17:08:02 DEBUG { 17:08:02 DEBUG }, 17:08:02 DEBUG "result": [ 17:08:02 DEBUG { 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "metric": { 17:08:02 DEBUG 17:08:02 DEBUG "pod": "ds-cts-0" 17:08:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:08:02 DEBUG "metric": { 17:08:02 DEBUG }, 17:08:02 DEBUG }, 17:08:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:08:02 DEBUG 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "value": [ 17:08:02 DEBUG { 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "pod": "ds-cts-0" 17:08:02 DEBUG "pod": "ds-idrepo-0" 17:08:02 DEBUG }, 17:08:02 DEBUG "pod": "ds-idrepo-1" 17:08:02 DEBUG }, 17:08:02 DEBUG "pod": "ds-idrepo-1" 17:08:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:08:02 DEBUG 17:08:02 DEBUG }, 17:08:02 DEBUG }, 17:08:02 DEBUG "pod": "ds-idrepo-0" 17:08:02 DEBUG "value": [ 17:08:02 DEBUG "value": [ 17:08:02 DEBUG }, 17:08:02 DEBUG 17:08:02 DEBUG "instance": "10.142.0.107:9100" 17:08:02 DEBUG 17:08:02 DEBUG "metric": { 17:08:02 DEBUG "pod": "ds-idrepo-0" 17:08:02 DEBUG }, 17:08:02 DEBUG }, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG }, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG }, 17:08:02 DEBUG }, 17:08:02 DEBUG 17:08:02 DEBUG "value": [ 17:08:02 DEBUG "value": [ 17:08:02 DEBUG }, 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG }, 17:08:02 DEBUG 17:08:02 DEBUG "instance": "10.142.0.107:9100" 17:08:02 DEBUG }, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "value": [ 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG }, 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "instance": "10.142.0.43:9100" 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG }, 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "instance": "10.142.0.43:9100" 17:08:02 DEBUG }, 17:08:02 DEBUG }, 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG }, 17:08:02 DEBUG [...] 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:08:02 DEBUG 17:08:02 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "value": [ 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG [...] 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG "pod": "ds-cts-2" 17:08:02 DEBUG 17:08:02 DEBUG }, 17:08:02 DEBUG 17:08:02 DEBUG }, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:08:02 DEBUG "pod": "ds-cts-2" 17:08:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:08:02 DEBUG 17:08:02 DEBUG "pod": "ds-cts-2" 17:08:02 DEBUG 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG 17:08:02 DEBUG 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG 17:08:02 DEBUG "pod": "ds-idrepo-2" 17:08:02 DEBUG }, 17:08:02 DEBUG 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG }, 17:08:02 DEBUG }, 17:08:02 DEBUG }, 17:08:02 DEBUG "pod": "ds-idrepo-2" 17:08:02 DEBUG }, 17:08:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:08:02 DEBUG "56865352.733333334" 17:08:02 DEBUG "1647479669760" 17:08:02 DEBUG "instance": "10.142.0.43:9100" 17:08:02 DEBUG 17:08:02 DEBUG "1087253.5" 17:08:02 DEBUG 17:08:02 DEBUG }, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG "0.000013085315511648814" 17:08:02 DEBUG "value": [ 17:08:02 DEBUG "value": [ 17:08:02 DEBUG "value": [ 17:08:02 DEBUG }, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG }, 17:08:02 DEBUG ] 17:08:02 DEBUG ] 17:08:02 DEBUG }, 17:08:02 DEBUG 17:08:02 DEBUG ] 17:08:02 DEBUG "pod": "ds-idrepo-2" 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG }, 17:08:02 DEBUG "961912832" 17:08:02 DEBUG }, 17:08:02 DEBUG "0" 17:08:02 DEBUG ] 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG "value": [ 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG "value": [ 17:08:02 DEBUG "instance": "10.142.0.43:9100" 17:08:02 DEBUG } 17:08:02 DEBUG }, 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG "0" 17:08:02 DEBUG "value": [ 17:08:02 DEBUG ] 17:08:02 DEBUG "value": [ 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG "0" 17:08:02 DEBUG "0" 17:08:02 DEBUG "7.851908603486872" 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG "0.000043097777777777775" 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG ] 17:08:02 DEBUG ] 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG }, 17:08:02 DEBUG ] 17:08:02 DEBUG "value": [ 17:08:02 DEBUG "0" 17:08:02 DEBUG ] 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG } 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG ] 17:08:02 DEBUG ] 17:08:02 DEBUG ] 17:08:02 DEBUG "0" 17:08:02 DEBUG ] 17:08:02 DEBUG "2337.1666666666665" 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG "0.056866666666730716" 17:08:02 DEBUG "value": [ 17:08:02 DEBUG } 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG "0.008" 17:08:02 DEBUG ] 17:08:02 DEBUG "2337.266666666667" 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG 1713542882, 17:08:02 DEBUG } 17:08:02 DEBUG "0" 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG ] 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG "56872866.86666667" 17:08:02 DEBUG ] 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG ] 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG ] 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:08:02 DEBUG } 17:09:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542941" 17:09:01 INFO [http_cmd]: http status code OK 17:09:01 DEBUG --- status code --- 17:09:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:01 DEBUG --- http response --- 17:09:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:09:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713542941" 17:09:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542941" 17:09:01 ERROR [http_cmd]: ERROR 17:09:01 ERROR --- status code --- 17:09:01 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:09:01 ERROR --- http response --- 17:09:01 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:09:01 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:09:01 INFO [http_cmd]: http status code OK 17:09:01 DEBUG --- status code --- 17:09:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:01 DEBUG --- http response --- 17:09:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:09:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713542941" 17:09:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542941" 17:09:01 INFO [http_cmd]: http status code OK 17:09:01 DEBUG --- status code --- 17:09:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:01 DEBUG --- http response --- 17:09:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:09:01 DEBUG ----- output ----- 17:09:01 DEBUG { 17:09:01 DEBUG "status": "success", 17:09:01 DEBUG "data": { 17:09:01 DEBUG "resultType": "vector", 17:09:01 DEBUG "result": [ 17:09:01 DEBUG { 17:09:01 DEBUG "metric": { 17:09:01 DEBUG "pod": "ds-idrepo-0" 17:09:01 DEBUG }, 17:09:01 DEBUG "value": [ 17:09:01 DEBUG 17:09:01 DEBUG 17:09:01 DEBUG 17:09:01 DEBUG [...] 17:09:01 DEBUG 17:09:01 DEBUG 17:09:01 DEBUG 17:09:01 DEBUG "pod": "ds-cts-2" 17:09:01 DEBUG }, 17:09:01 DEBUG "value": [ 17:09:01 DEBUG 1713542941, 17:09:01 DEBUG "0" 17:09:01 DEBUG ] 17:09:01 DEBUG } 17:09:01 DEBUG ] 17:09:01 DEBUG } 17:09:01 DEBUG } 17:09:01 INFO [http_cmd]: http status code OK 17:09:01 DEBUG --- status code --- 17:09:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:01 DEBUG --- http response --- 17:09:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:09:01 DEBUG ----- output ----- 17:09:01 DEBUG { 17:09:01 DEBUG "status": "success", 17:09:01 DEBUG "data": { 17:09:01 DEBUG "resultType": "vector", 17:09:01 DEBUG "result": [ 17:09:01 DEBUG { 17:09:01 DEBUG "metric": { 17:09:01 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:09:01 DEBUG }, 17:09:01 DEBUG "value": [ 17:09:01 DEBUG 17:09:01 DEBUG 17:09:01 DEBUG 17:09:01 DEBUG [...] 17:09:01 DEBUG 17:09:01 DEBUG 17:09:01 DEBUG 17:09:01 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:09:01 DEBUG }, 17:09:01 DEBUG "value": [ 17:09:01 DEBUG 1713542941, 17:09:01 DEBUG "2330.366666666667" 17:09:01 DEBUG ] 17:09:01 DEBUG } 17:09:01 DEBUG ] 17:09:01 DEBUG } 17:09:01 DEBUG } 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG { 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "data": { 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "result": [ 17:09:02 DEBUG { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG "0" 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542942" 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG { 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "data": { 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "result": [ 17:09:02 DEBUG { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "instance": "10.142.0.107:9100" 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG "instance": "10.142.0.43:9100" 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG "0.0396666666666628" 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG { 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "data": { 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "result": [ 17:09:02 DEBUG { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG "pod": "ds-idrepo-2" 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG "152327201.1211226" 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 INFO 17:09:02 INFO [loop_until]: kubectl --namespace=xlou top node 17:09:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:09:02 INFO 17:09:02 INFO [loop_until]: kubectl --namespace=xlou top pods 17:09:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:09:02 INFO [loop_until]: OK (rc = 0) 17:09:02 DEBUG --- stdout --- 17:09:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6179Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5398m 18% 6976Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 47m 0% 2672Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11069m 37% 11966Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 8149m 27% 15434Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5227Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11355m 38% 11611Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12275m 41% 2958Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12906m 43% 10972Mi 9% gke-xlou-cdm-ds-0f601751-j488 6234m 20% 6612Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 8419m 28% 17169Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6382m 21% 7170Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 8350m 27% 14886Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8060m 50% 4630Mi 7% 17:09:02 DEBUG --- stderr --- 17:09:02 DEBUG 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713542942" 17:09:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713542942" 17:09:02 INFO [loop_until]: OK (rc = 0) 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 DEBUG --- status code --- 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 DEBUG --- stdout --- 17:09:02 DEBUG --- status code --- 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 DEBUG --- status code --- 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 DEBUG --- status code --- 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG --- status code --- 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- status code --- 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 DEBUG --- status code --- 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11197m 8689Mi am-6fd554f8f7-btd4s 11044m 8755Mi am-6fd554f8f7-wxcsj 11824m 8763Mi ds-cts-0 7989m 12380Mi ds-cts-1 8218m 12314Mi ds-cts-2 8516m 12371Mi ds-idrepo-0 5617m 3421Mi ds-idrepo-1 5452m 3356Mi ds-idrepo-2 5479m 3409Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7750m 827Mi 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG --- status code --- 17:09:02 INFO [http_cmd]: http status code OK 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG --- stderr --- 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- status code --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:09:02 DEBUG --- http response --- 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:09:02 DEBUG { 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG { 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG { 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG { 17:09:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG { 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG { 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "data": { 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG { 17:09:02 DEBUG "data": { 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "data": { 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "data": { 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG ----- output ----- 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "data": { 17:09:02 DEBUG "data": { 17:09:02 DEBUG "data": { 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "data": { 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "data": { 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "data": { 17:09:02 DEBUG "data": { 17:09:02 DEBUG "data": { 17:09:02 DEBUG "data": { 17:09:02 DEBUG "result": [ 17:09:02 DEBUG { 17:09:02 DEBUG "data": { 17:09:02 DEBUG "data": { 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "data": { 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "data": { 17:09:02 DEBUG "data": { 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG { 17:09:02 DEBUG "status": "success", 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG { 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG "result": [ 17:09:02 DEBUG { 17:09:02 DEBUG "result": [ 17:09:02 DEBUG { 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "data": { 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "result": [ 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "result": [ 17:09:02 DEBUG { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:09:02 DEBUG "resultType": "vector", 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG { 17:09:02 DEBUG "pod": "ds-cts-0" 17:09:02 DEBUG { 17:09:02 DEBUG { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "instance": "10.142.0.107:9100" 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG }, 17:09:02 DEBUG "result": [ 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "instance": "10.142.0.107:9100" 17:09:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:09:02 DEBUG "pod": "ds-cts-0" 17:09:02 DEBUG "metric": { 17:09:02 DEBUG }, 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "metric": { 17:09:02 DEBUG "instance": "10.142.0.107:9100" 17:09:02 DEBUG }, 17:09:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:09:02 DEBUG }, 17:09:02 DEBUG "instance": "10.142.0.107:9100" 17:09:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:09:02 DEBUG "pod": "ds-idrepo-0" 17:09:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:09:02 DEBUG "value": [ 17:09:02 DEBUG { 17:09:02 DEBUG "pod": "ds-idrepo-0" 17:09:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:09:02 DEBUG }, 17:09:02 DEBUG }, 17:09:02 DEBUG }, 17:09:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "pod": "ds-idrepo-0" 17:09:02 DEBUG "pod": "ds-cts-0" 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG }, 17:09:02 DEBUG }, 17:09:02 DEBUG }, 17:09:02 DEBUG }, 17:09:02 DEBUG 17:09:02 DEBUG "metric": { 17:09:02 DEBUG }, 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "value": [ 17:09:02 DEBUG }, 17:09:02 DEBUG 17:09:02 DEBUG }, 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 17:09:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG }, 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG [...] 17:09:02 DEBUG [...] 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG [...] 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG [...] 17:09:02 DEBUG 17:09:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG "pod": "ds-idrepo-2" 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG "instance": "10.142.0.43:9100" 17:09:02 DEBUG 17:09:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:09:02 DEBUG }, 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG }, 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:09:02 DEBUG "pod": "ds-idrepo-2" 17:09:02 DEBUG "pod": "ds-cts-2" 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "pod": "ds-idrepo-2" 17:09:02 DEBUG 17:09:02 DEBUG "instance": "10.142.0.43:9100" 17:09:02 DEBUG }, 17:09:02 DEBUG 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:09:02 DEBUG "pod": "ds-idrepo-2" 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:09:02 DEBUG 17:09:02 DEBUG 17:09:02 DEBUG "pod": "ds-idrepo-2" 17:09:02 DEBUG }, 17:09:02 DEBUG }, 17:09:02 DEBUG "pod": "ds-cts-2" 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG 17:09:02 DEBUG }, 17:09:02 DEBUG "pod": "ds-cts-2" 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG }, 17:09:02 DEBUG }, 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG }, 17:09:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:09:02 DEBUG "instance": "10.142.0.43:9100" 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "value": [ 17:09:02 DEBUG }, 17:09:02 DEBUG "0" 17:09:02 DEBUG "pod": "ds-idrepo-2" 17:09:02 DEBUG "value": [ 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG "instance": "10.142.0.43:9100" 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG "1227328.5" 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "0" 17:09:02 DEBUG "value": [ 17:09:02 DEBUG }, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG ] 17:09:02 DEBUG }, 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG "value": [ 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG "1650138598400" 17:09:02 DEBUG }, 17:09:02 DEBUG "2330.2666666666664" 17:09:02 DEBUG ] 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG ] 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG }, 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG "7.87154521445745" 17:09:02 DEBUG "3.2853202273959883" 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG } 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "4182.90325231616" 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG "0.05719999999991027" 17:09:02 DEBUG ] 17:09:02 DEBUG "value": [ 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG "2330.3" 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG "962957312" 17:09:02 DEBUG "value": [ 17:09:02 DEBUG "0.008" 17:09:02 DEBUG "0" 17:09:02 DEBUG ] 17:09:02 DEBUG ] 17:09:02 DEBUG "0.03333333333333333" 17:09:02 DEBUG ] 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG ] 17:09:02 DEBUG 1713542942, 17:09:02 DEBUG ] 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG "0" 17:09:02 DEBUG "0.000021548888888888887" 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG ] 17:09:02 DEBUG "0" 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG "56641717.03333333" 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG "56648556.43333334" 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG } 17:09:02 DEBUG ] 17:09:02 DEBUG } 17:09:02 DEBUG } 17:10:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543001" 17:10:01 INFO [http_cmd]: http status code OK 17:10:01 DEBUG --- status code --- 17:10:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:01 DEBUG --- http response --- 17:10:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:10:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543001" 17:10:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543001" 17:10:01 ERROR [http_cmd]: ERROR 17:10:01 ERROR --- status code --- 17:10:01 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:10:01 ERROR --- http response --- 17:10:01 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:10:01 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:10:01 INFO [http_cmd]: http status code OK 17:10:01 DEBUG --- status code --- 17:10:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:01 DEBUG --- http response --- 17:10:01 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG { 17:10:02 DEBUG { 17:10:02 DEBUG "status": "success", 17:10:02 DEBUG "status": "success", 17:10:02 DEBUG "data": { 17:10:02 DEBUG "data": { 17:10:02 DEBUG "resultType": "vector", 17:10:02 DEBUG "resultType": "vector", 17:10:02 DEBUG "result": [ 17:10:02 DEBUG "result": [ 17:10:02 DEBUG { 17:10:02 DEBUG { 17:10:02 DEBUG "metric": { 17:10:02 DEBUG "metric": { 17:10:02 DEBUG "pod": "ds-idrepo-0" 17:10:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:10:02 DEBUG }, 17:10:02 DEBUG }, 17:10:02 DEBUG "value": [ 17:10:02 DEBUG "value": [ 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG [...] 17:10:02 DEBUG [...] 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG "pod": "ds-cts-2" 17:10:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:10:02 DEBUG }, 17:10:02 DEBUG }, 17:10:02 DEBUG "value": [ 17:10:02 DEBUG "value": [ 17:10:02 DEBUG 1713543002, 17:10:02 DEBUG 1713543002, 17:10:02 DEBUG "0" 17:10:02 DEBUG "2328.9666666666667" 17:10:02 DEBUG ] 17:10:02 DEBUG ] 17:10:02 DEBUG } 17:10:02 DEBUG } 17:10:02 DEBUG ] 17:10:02 DEBUG ] 17:10:02 DEBUG } 17:10:02 DEBUG } 17:10:02 DEBUG } 17:10:02 DEBUG } 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG { 17:10:02 DEBUG "status": "success", 17:10:02 DEBUG "data": { 17:10:02 DEBUG "resultType": "vector", 17:10:02 DEBUG "result": [ 17:10:02 DEBUG { 17:10:02 DEBUG "metric": { 17:10:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:10:02 DEBUG }, 17:10:02 DEBUG "value": [ 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG [...] 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:10:02 DEBUG }, 17:10:02 DEBUG "value": [ 17:10:02 DEBUG 1713543002, 17:10:02 DEBUG "0" 17:10:02 DEBUG ] 17:10:02 DEBUG } 17:10:02 DEBUG ] 17:10:02 DEBUG } 17:10:02 DEBUG } 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543002" 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG { 17:10:02 DEBUG "status": "success", 17:10:02 DEBUG "data": { 17:10:02 DEBUG "resultType": "vector", 17:10:02 DEBUG "result": [ 17:10:02 DEBUG { 17:10:02 DEBUG "metric": { 17:10:02 DEBUG "instance": "10.142.0.107:9100" 17:10:02 DEBUG }, 17:10:02 DEBUG "value": [ 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG [...] 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG "instance": "10.142.0.43:9100" 17:10:02 DEBUG }, 17:10:02 DEBUG "value": [ 17:10:02 DEBUG 1713543002, 17:10:02 DEBUG "0.038666666666661825" 17:10:02 DEBUG ] 17:10:02 DEBUG } 17:10:02 DEBUG ] 17:10:02 DEBUG } 17:10:02 DEBUG } 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG { 17:10:02 DEBUG "status": "success", 17:10:02 DEBUG "data": { 17:10:02 DEBUG "resultType": "vector", 17:10:02 DEBUG "result": [ 17:10:02 DEBUG { 17:10:02 DEBUG "metric": { 17:10:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:10:02 DEBUG }, 17:10:02 DEBUG "value": [ 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG [...] 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:10:02 DEBUG }, 17:10:02 DEBUG "value": [ 17:10:02 DEBUG 1713543002, 17:10:02 DEBUG "0" 17:10:02 DEBUG ] 17:10:02 DEBUG } 17:10:02 DEBUG ] 17:10:02 DEBUG } 17:10:02 DEBUG } 17:10:02 INFO 17:10:02 INFO [loop_until]: kubectl --namespace=xlou top node 17:10:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO 17:10:02 INFO [loop_until]: kubectl --namespace=xlou top pods 17:10:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG { 17:10:02 DEBUG "status": "success", 17:10:02 DEBUG "data": { 17:10:02 DEBUG "resultType": "vector", 17:10:02 DEBUG "result": [ 17:10:02 DEBUG { 17:10:02 DEBUG "metric": { 17:10:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:10:02 DEBUG }, 17:10:02 DEBUG "value": [ 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG [...] 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG 17:10:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:10:02 DEBUG }, 17:10:02 DEBUG "value": [ 17:10:02 DEBUG 1713543002, 17:10:02 DEBUG "1367227" 17:10:02 DEBUG ] 17:10:02 DEBUG } 17:10:02 DEBUG ] 17:10:02 DEBUG } 17:10:02 DEBUG } 17:10:02 INFO [loop_until]: OK (rc = 0) 17:10:02 DEBUG --- stdout --- 17:10:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6182Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5438m 18% 6991Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 48m 0% 2674Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 10723m 35% 11952Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 8635m 28% 15434Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5214Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11276m 37% 11620Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12436m 41% 2958Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12902m 43% 10985Mi 9% gke-xlou-cdm-ds-0f601751-j488 6216m 20% 6611Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 8800m 29% 17167Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6370m 21% 7159Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 8927m 29% 14881Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8045m 50% 4630Mi 7% 17:10:02 DEBUG --- stderr --- 17:10:02 DEBUG 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543002" 17:10:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543002" 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 INFO [loop_until]: OK (rc = 0) 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG --- stdout --- 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG --- status code --- 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG --- status code --- 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- status code --- 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- status code --- 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11181m 8689Mi am-6fd554f8f7-btd4s 11356m 8756Mi am-6fd554f8f7-wxcsj 11224m 8764Mi ds-cts-0 8676m 12386Mi ds-cts-1 8836m 12321Mi ds-cts-2 8977m 12379Mi ds-idrepo-0 5825m 3421Mi ds-idrepo-1 5463m 3357Mi ds-idrepo-2 5579m 3401Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7650m 829Mi 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- status code --- 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- stderr --- 17:10:02 DEBUG 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:10:02 DEBUG --- http response --- 17:10:02 INFO [http_cmd]: http status code OK 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG --- status code --- 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG --- http response --- 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG { 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:10:02 DEBUG { 17:10:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:10:02 DEBUG { 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG { 17:10:02 DEBUG { 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG { 17:10:02 DEBUG ----- output ----- 17:10:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:10:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:10:03 DEBUG { 17:10:03 DEBUG ----- output ----- 17:10:03 DEBUG ----- output ----- 17:10:03 DEBUG { 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG { 17:10:03 DEBUG ----- output ----- 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG --- http response --- 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG { 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG { 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG { 17:10:03 DEBUG ----- output ----- 17:10:03 DEBUG ----- output ----- 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG { 17:10:03 DEBUG { 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG "data": { 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG { 17:10:03 DEBUG "data": { 17:10:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:10:03 DEBUG "data": { 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG "data": { 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG "data": { 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG { 17:10:03 DEBUG { 17:10:03 DEBUG "data": { 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG "data": { 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "data": { 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG ----- output ----- 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "data": { 17:10:03 DEBUG "result": [ 17:10:03 DEBUG "data": { 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "data": { 17:10:03 DEBUG "data": { 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "data": { 17:10:03 DEBUG "data": { 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "result": [ 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "data": { 17:10:03 DEBUG "result": [ 17:10:03 DEBUG { 17:10:03 DEBUG "result": [ 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG { 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "result": [ 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "data": { 17:10:03 DEBUG "data": { 17:10:03 DEBUG "result": [ 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "result": [ 17:10:03 DEBUG { 17:10:03 DEBUG "result": [ 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG { 17:10:03 DEBUG "status": "success", 17:10:03 DEBUG { 17:10:03 DEBUG "result": [ 17:10:03 DEBUG "metric": { 17:10:03 DEBUG "result": [ 17:10:03 DEBUG { 17:10:03 DEBUG "result": [ 17:10:03 DEBUG "result": [ 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "result": [ 17:10:03 DEBUG "result": [ 17:10:03 DEBUG "result": [ 17:10:03 DEBUG { 17:10:03 DEBUG "metric": { 17:10:03 DEBUG { 17:10:03 DEBUG "result": [ 17:10:03 DEBUG "metric": { 17:10:03 DEBUG "data": { 17:10:03 DEBUG "metric": { 17:10:03 DEBUG { 17:10:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:10:03 DEBUG { 17:10:03 DEBUG "metric": { 17:10:03 DEBUG { 17:10:03 DEBUG { 17:10:03 DEBUG "result": [ 17:10:03 DEBUG { 17:10:03 DEBUG { 17:10:03 DEBUG { 17:10:03 DEBUG { 17:10:03 DEBUG "metric": { 17:10:03 DEBUG "pod": "ds-cts-0" 17:10:03 DEBUG "metric": { 17:10:03 DEBUG { 17:10:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:10:03 DEBUG "resultType": "vector", 17:10:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:10:03 DEBUG "metric": { 17:10:03 DEBUG }, 17:10:03 DEBUG "metric": { 17:10:03 DEBUG "instance": "10.142.0.107:9100" 17:10:03 DEBUG "metric": { 17:10:03 DEBUG "metric": { 17:10:03 DEBUG { 17:10:03 DEBUG "metric": { 17:10:03 DEBUG "metric": { 17:10:03 DEBUG "metric": { 17:10:03 DEBUG "metric": { 17:10:03 DEBUG "instance": "10.142.0.107:9100" 17:10:03 DEBUG }, 17:10:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:10:03 DEBUG "metric": { 17:10:03 DEBUG }, 17:10:03 DEBUG "result": [ 17:10:03 DEBUG }, 17:10:03 DEBUG "pod": "ds-cts-0" 17:10:03 DEBUG "value": [ 17:10:03 DEBUG "pod": "ds-idrepo-0" 17:10:03 DEBUG }, 17:10:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:10:03 DEBUG "pod": "ds-idrepo-0" 17:10:03 DEBUG "metric": { 17:10:03 DEBUG "instance": "10.142.0.107:9100" 17:10:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:10:03 DEBUG "pod": "ds-idrepo-0" 17:10:03 DEBUG "pod": "ds-cts-0" 17:10:03 DEBUG }, 17:10:03 DEBUG "value": [ 17:10:03 DEBUG }, 17:10:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:10:03 DEBUG "value": [ 17:10:03 DEBUG { 17:10:03 DEBUG "value": [ 17:10:03 DEBUG }, 17:10:03 DEBUG 17:10:03 DEBUG }, 17:10:03 DEBUG "value": [ 17:10:03 DEBUG }, 17:10:03 DEBUG }, 17:10:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:10:03 DEBUG }, 17:10:03 DEBUG }, 17:10:03 DEBUG }, 17:10:03 DEBUG }, 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 17:10:03 DEBUG "value": [ 17:10:03 DEBUG }, 17:10:03 DEBUG 17:10:03 DEBUG "metric": { 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 17:10:03 DEBUG "value": [ 17:10:03 DEBUG "value": [ 17:10:03 DEBUG }, 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 17:10:03 DEBUG "value": [ 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 17:10:03 DEBUG "instance": "10.142.0.107:9100" 17:10:03 DEBUG 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG }, 17:10:03 DEBUG [...] 17:10:03 DEBUG 17:10:03 DEBUG [...] 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG [...] 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG [...] 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG [...] 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG [...] 17:10:03 DEBUG 17:10:03 DEBUG [...] 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG [...] 17:10:03 DEBUG 17:10:03 DEBUG [...] 17:10:03 DEBUG [...] 17:10:03 DEBUG [...] 17:10:03 DEBUG [...] 17:10:03 DEBUG 17:10:03 DEBUG [...] 17:10:03 DEBUG [...] 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG [...] 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG [...] 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG [...] 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:10:03 DEBUG 17:10:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG "pod": "ds-cts-2" 17:10:03 DEBUG }, 17:10:03 DEBUG 17:10:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:10:03 DEBUG [...] 17:10:03 DEBUG }, 17:10:03 DEBUG 17:10:03 DEBUG }, 17:10:03 DEBUG "pod": "ds-idrepo-2" 17:10:03 DEBUG 17:10:03 DEBUG "pod": "ds-idrepo-2" 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG "instance": "10.142.0.43:9100" 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:10:03 DEBUG 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG }, 17:10:03 DEBUG 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 17:10:03 DEBUG "value": [ 17:10:03 DEBUG }, 17:10:03 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:10:03 DEBUG }, 17:10:03 DEBUG 17:10:03 DEBUG 17:10:03 DEBUG }, 17:10:03 DEBUG 17:10:03 DEBUG "instance": "10.142.0.43:9100" 17:10:03 DEBUG }, 17:10:03 DEBUG }, 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:10:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG "0" 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG "value": [ 17:10:03 DEBUG }, 17:10:03 DEBUG "value": [ 17:10:03 DEBUG "instance": "10.142.0.43:9100" 17:10:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:10:03 DEBUG }, 17:10:03 DEBUG "pod": "ds-idrepo-2" 17:10:03 DEBUG "pod": "ds-cts-2" 17:10:03 DEBUG "value": [ 17:10:03 DEBUG "value": [ 17:10:03 DEBUG "0" 17:10:03 DEBUG }, 17:10:03 DEBUG }, 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG 17:10:03 DEBUG "pod": "ds-cts-2" 17:10:03 DEBUG }, 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG }, 17:10:03 DEBUG "value": [ 17:10:03 DEBUG "value": [ 17:10:03 DEBUG }, 17:10:03 DEBUG }, 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG ] 17:10:03 DEBUG "value": [ 17:10:03 DEBUG "value": [ 17:10:03 DEBUG "7.912325813184271" 17:10:03 DEBUG "instance": "10.142.0.43:9100" 17:10:03 DEBUG ] 17:10:03 DEBUG "2329.0333333333333" 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG "0.000021548888888888887" 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG "0" 17:10:03 DEBUG "value": [ 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG "value": [ 17:10:03 DEBUG "value": [ 17:10:03 DEBUG "0.056799999999969944" 17:10:03 DEBUG "2329.133333333333" 17:10:03 DEBUG } 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG "0.0000066715430320334254" 17:10:03 DEBUG ] 17:10:03 DEBUG }, 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG "0" 17:10:03 DEBUG ] 17:10:03 DEBUG "0" 17:10:03 DEBUG ] 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG "1652845306880" 17:10:03 DEBUG "964677632" 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG ] 17:10:03 DEBUG ] 17:10:03 DEBUG ] 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG "value": [ 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG "56484803.63333333" 17:10:03 DEBUG ] 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG "0" 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG "0.0095" 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG 1713543002, 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG "0" 17:10:03 DEBUG ] 17:10:03 DEBUG ] 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG "56477995.9" 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG ] 17:10:03 DEBUG ] 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:10:03 DEBUG } 17:11:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543061" 17:11:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543061" 17:11:02 INFO [http_cmd]: http status code OK 17:11:02 DEBUG --- status code --- 17:11:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:02 DEBUG --- http response --- 17:11:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:11:02 INFO [http_cmd]: http status code OK 17:11:02 DEBUG --- status code --- 17:11:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:02 DEBUG --- http response --- 17:11:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:11:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543062" 17:11:02 ERROR [http_cmd]: ERROR 17:11:02 ERROR --- status code --- 17:11:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:11:02 ERROR --- http response --- 17:11:02 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:11:02 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:11:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543062" 17:11:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543062" 17:11:02 INFO [http_cmd]: http status code OK 17:11:02 DEBUG --- status code --- 17:11:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:02 DEBUG --- http response --- 17:11:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:11:02 DEBUG ----- output ----- 17:11:02 DEBUG { 17:11:02 DEBUG "status": "success", 17:11:02 DEBUG "data": { 17:11:02 DEBUG "resultType": "vector", 17:11:02 DEBUG "result": [ 17:11:02 DEBUG { 17:11:02 DEBUG "metric": { 17:11:02 DEBUG "pod": "ds-idrepo-0" 17:11:02 DEBUG }, 17:11:02 DEBUG "value": [ 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG [...] 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG "pod": "ds-cts-2" 17:11:02 DEBUG }, 17:11:02 DEBUG "value": [ 17:11:02 DEBUG 1713543062, 17:11:02 DEBUG "0" 17:11:02 DEBUG ] 17:11:02 DEBUG } 17:11:02 DEBUG ] 17:11:02 DEBUG } 17:11:02 DEBUG } 17:11:02 INFO [http_cmd]: http status code OK 17:11:02 DEBUG --- status code --- 17:11:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:02 DEBUG --- http response --- 17:11:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:11:02 DEBUG ----- output ----- 17:11:02 DEBUG { 17:11:02 DEBUG "status": "success", 17:11:02 DEBUG "data": { 17:11:02 DEBUG "resultType": "vector", 17:11:02 DEBUG "result": [ 17:11:02 DEBUG { 17:11:02 DEBUG "metric": { 17:11:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:11:02 DEBUG }, 17:11:02 DEBUG "value": [ 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG [...] 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:11:02 DEBUG }, 17:11:02 DEBUG "value": [ 17:11:02 DEBUG 1713543062, 17:11:02 DEBUG "2330" 17:11:02 DEBUG ] 17:11:02 DEBUG } 17:11:02 DEBUG ] 17:11:02 DEBUG } 17:11:02 DEBUG } 17:11:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543062" 17:11:02 INFO [http_cmd]: http status code OK 17:11:02 DEBUG --- status code --- 17:11:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:02 DEBUG --- http response --- 17:11:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:11:02 DEBUG ----- output ----- 17:11:02 DEBUG { 17:11:02 DEBUG "status": "success", 17:11:02 DEBUG "data": { 17:11:02 DEBUG "resultType": "vector", 17:11:02 DEBUG "result": [ 17:11:02 DEBUG { 17:11:02 DEBUG "metric": { 17:11:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:11:02 DEBUG }, 17:11:02 DEBUG "value": [ 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG [...] 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:11:02 DEBUG }, 17:11:02 DEBUG "value": [ 17:11:02 DEBUG 1713543062, 17:11:02 DEBUG "0" 17:11:02 DEBUG ] 17:11:02 DEBUG } 17:11:02 DEBUG ] 17:11:02 DEBUG } 17:11:02 DEBUG } 17:11:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543062" 17:11:02 INFO [http_cmd]: http status code OK 17:11:02 DEBUG --- status code --- 17:11:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:02 DEBUG --- http response --- 17:11:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:11:02 DEBUG ----- output ----- 17:11:02 DEBUG { 17:11:02 DEBUG "status": "success", 17:11:02 DEBUG "data": { 17:11:02 DEBUG "resultType": "vector", 17:11:02 DEBUG "result": [ 17:11:02 DEBUG { 17:11:02 DEBUG "metric": { 17:11:02 DEBUG "instance": "10.142.0.107:9100" 17:11:02 DEBUG }, 17:11:02 DEBUG "value": [ 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG [...] 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG "instance": "10.142.0.43:9100" 17:11:02 DEBUG }, 17:11:02 DEBUG "value": [ 17:11:02 DEBUG 1713543062, 17:11:02 DEBUG "0.038666666666663706" 17:11:02 DEBUG ] 17:11:02 DEBUG } 17:11:02 DEBUG ] 17:11:02 DEBUG } 17:11:02 DEBUG } 17:11:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543062" 17:11:02 INFO [http_cmd]: http status code OK 17:11:02 DEBUG --- status code --- 17:11:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:02 DEBUG --- http response --- 17:11:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 17:11:02 DEBUG ----- output ----- 17:11:02 DEBUG { 17:11:02 DEBUG "status": "success", 17:11:02 DEBUG "data": { 17:11:02 DEBUG "resultType": "vector", 17:11:02 DEBUG "result": [ 17:11:02 DEBUG { 17:11:02 DEBUG "metric": { 17:11:02 DEBUG "pod": "ds-idrepo-1" 17:11:02 DEBUG }, 17:11:02 DEBUG "value": [ 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG [...] 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:11:02 DEBUG }, 17:11:02 DEBUG "value": [ 17:11:02 DEBUG 1713543062, 17:11:02 DEBUG "0" 17:11:02 DEBUG ] 17:11:02 DEBUG } 17:11:02 DEBUG ] 17:11:02 DEBUG } 17:11:02 DEBUG } 17:11:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543062" 17:11:02 INFO 17:11:02 INFO [loop_until]: kubectl --namespace=xlou top node 17:11:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:11:02 INFO [http_cmd]: http status code OK 17:11:02 DEBUG --- status code --- 17:11:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:02 DEBUG --- http response --- 17:11:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:11:02 DEBUG ----- output ----- 17:11:02 DEBUG { 17:11:02 DEBUG "status": "success", 17:11:02 DEBUG "data": { 17:11:02 DEBUG "resultType": "vector", 17:11:02 DEBUG "result": [ 17:11:02 DEBUG { 17:11:02 DEBUG "metric": { 17:11:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:11:02 DEBUG }, 17:11:02 DEBUG "value": [ 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG [...] 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG 17:11:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:11:02 DEBUG }, 17:11:02 DEBUG "value": [ 17:11:02 DEBUG 1713543062, 17:11:02 DEBUG "1507146.5" 17:11:02 DEBUG ] 17:11:02 DEBUG } 17:11:02 DEBUG ] 17:11:02 DEBUG } 17:11:02 DEBUG } 17:11:03 INFO [loop_until]: OK (rc = 0) 17:11:03 DEBUG --- stdout --- 17:11:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6178Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5204m 17% 6984Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 46m 0% 2667Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 10897m 36% 11960Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 8734m 29% 15451Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5234Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11202m 37% 11624Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 11642m 38% 2938Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12647m 42% 10975Mi 9% gke-xlou-cdm-ds-0f601751-j488 6196m 20% 6621Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 8464m 28% 17172Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6319m 21% 7153Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 8853m 29% 14874Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8003m 50% 4631Mi 7% 17:11:03 DEBUG --- stderr --- 17:11:03 DEBUG 17:11:03 INFO 17:11:03 INFO [loop_until]: kubectl --namespace=xlou top pods 17:11:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543063" 17:11:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543063" 17:11:03 INFO [loop_until]: OK (rc = 0) 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 DEBUG --- status code --- 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 DEBUG --- stdout --- 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG --- status code --- 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG --- status code --- 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG --- status code --- 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11138m 8690Mi am-6fd554f8f7-btd4s 11004m 8758Mi am-6fd554f8f7-wxcsj 11678m 8765Mi ds-cts-0 8644m 12392Mi ds-cts-1 8734m 12326Mi ds-cts-2 8658m 12385Mi ds-idrepo-0 5558m 3421Mi ds-idrepo-1 5452m 3356Mi ds-idrepo-2 5440m 3409Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7757m 831Mi 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG --- stderr --- 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG --- http response --- 17:11:03 INFO [http_cmd]: http status code OK 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:11:03 DEBUG --- status code --- 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG { 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG { 17:11:03 DEBUG { 17:11:03 DEBUG { 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:11:03 DEBUG { 17:11:03 DEBUG --- http response --- 17:11:03 DEBUG { 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG { 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG { 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG { 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG { 17:11:03 DEBUG { 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG { 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG { 17:11:03 DEBUG { 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG { 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG { 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG "data": { 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG "data": { 17:11:03 DEBUG "data": { 17:11:03 DEBUG "data": { 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG { 17:11:03 DEBUG "data": { 17:11:03 DEBUG ----- output ----- 17:11:03 DEBUG "data": { 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG "data": { 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG "data": { 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG "data": { 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "data": { 17:11:03 DEBUG "data": { 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "data": { 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG { 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "data": { 17:11:03 DEBUG "data": { 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "data": { 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "data": { 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "status": "success", 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "result": [ 17:11:03 DEBUG { 17:11:03 DEBUG "result": [ 17:11:03 DEBUG { 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "result": [ 17:11:03 DEBUG { 17:11:03 DEBUG { 17:11:03 DEBUG { 17:11:03 DEBUG "data": { 17:11:03 DEBUG { 17:11:03 DEBUG { 17:11:03 DEBUG "data": { 17:11:03 DEBUG { 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "result": [ 17:11:03 DEBUG { 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "metric": { 17:11:03 DEBUG { 17:11:03 DEBUG "metric": { 17:11:03 DEBUG { 17:11:03 DEBUG { 17:11:03 DEBUG "metric": { 17:11:03 DEBUG "metric": { 17:11:03 DEBUG "metric": { 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "metric": { 17:11:03 DEBUG "metric": { 17:11:03 DEBUG "resultType": "vector", 17:11:03 DEBUG "metric": { 17:11:03 DEBUG { 17:11:03 DEBUG { 17:11:03 DEBUG "metric": { 17:11:03 DEBUG { 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "pod": "ds-idrepo-0" 17:11:03 DEBUG "metric": { 17:11:03 DEBUG "instance": "10.142.0.107:9100" 17:11:03 DEBUG "metric": { 17:11:03 DEBUG "metric": { 17:11:03 DEBUG "pod": "ds-idrepo-1" 17:11:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:11:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:11:03 DEBUG "result": [ 17:11:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:11:03 DEBUG "pod": "ds-cts-0" 17:11:03 DEBUG }, 17:11:03 DEBUG "pod": "ds-cts-0" 17:11:03 DEBUG "metric": { 17:11:03 DEBUG "metric": { 17:11:03 DEBUG "instance": "10.142.0.107:9100" 17:11:03 DEBUG "metric": { 17:11:03 DEBUG { 17:11:03 DEBUG }, 17:11:03 DEBUG "instance": "10.142.0.107:9100" 17:11:03 DEBUG }, 17:11:03 DEBUG "instance": "10.142.0.107:9100" 17:11:03 DEBUG "pod": "ds-idrepo-1" 17:11:03 DEBUG }, 17:11:03 DEBUG }, 17:11:03 DEBUG }, 17:11:03 DEBUG { 17:11:03 DEBUG }, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG "result": [ 17:11:03 DEBUG }, 17:11:03 DEBUG "pod": "ds-idrepo-1" 17:11:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:11:03 DEBUG }, 17:11:03 DEBUG "pod": "ds-idrepo-0" 17:11:03 DEBUG "metric": { 17:11:03 DEBUG "value": [ 17:11:03 DEBUG }, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG }, 17:11:03 DEBUG }, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG "value": [ 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 17:11:03 DEBUG "metric": { 17:11:03 DEBUG "pod": "ds-idrepo-0" 17:11:03 DEBUG { 17:11:03 DEBUG "value": [ 17:11:03 DEBUG }, 17:11:03 DEBUG }, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG }, 17:11:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:11:03 DEBUG 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 17:11:03 DEBUG "value": [ 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG }, 17:11:03 DEBUG "metric": { 17:11:03 DEBUG 17:11:03 DEBUG "value": [ 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 17:11:03 DEBUG "value": [ 17:11:03 DEBUG }, 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG "pod": "ds-cts-0" 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG [...] 17:11:03 DEBUG "value": [ 17:11:03 DEBUG }, 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG "value": [ 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:11:03 DEBUG 17:11:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:11:03 DEBUG "pod": "ds-cts-2" 17:11:03 DEBUG }, 17:11:03 DEBUG 17:11:03 DEBUG [...] 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG "pod": "ds-idrepo-2" 17:11:03 DEBUG 17:11:03 DEBUG "instance": "10.142.0.43:9100" 17:11:03 DEBUG 17:11:03 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:11:03 DEBUG 17:11:03 DEBUG }, 17:11:03 DEBUG 17:11:03 DEBUG }, 17:11:03 DEBUG [...] 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG "pod": "ds-cts-2" 17:11:03 DEBUG 17:11:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG }, 17:11:03 DEBUG "instance": "10.142.0.43:9100" 17:11:03 DEBUG }, 17:11:03 DEBUG "instance": "10.142.0.43:9100" 17:11:03 DEBUG }, 17:11:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG 17:11:03 DEBUG 17:11:03 DEBUG }, 17:11:03 DEBUG "instance": "10.142.0.43:9100" 17:11:03 DEBUG }, 17:11:03 DEBUG "pod": "ds-idrepo-2" 17:11:03 DEBUG 17:11:03 DEBUG "value": [ 17:11:03 DEBUG }, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG }, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG }, 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:11:03 DEBUG }, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:11:03 DEBUG 17:11:03 DEBUG "pod": "ds-cts-2" 17:11:03 DEBUG }, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG }, 17:11:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG "0" 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG 17:11:03 DEBUG "0" 17:11:03 DEBUG "0.0125" 17:11:03 DEBUG }, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG }, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG }, 17:11:03 DEBUG "0" 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "1655545705472" 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "0" 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG ] 17:11:03 DEBUG "7.873161777743995" 17:11:03 DEBUG 17:11:03 DEBUG ] 17:11:03 DEBUG ] 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "966475776" 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "value": [ 17:11:03 DEBUG ] 17:11:03 DEBUG "0.05653333333326884" 17:11:03 DEBUG ] 17:11:03 DEBUG "56561684.53333333" 17:11:03 DEBUG ] 17:11:03 DEBUG "2330.0333333333333" 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG "pod": "ds-idrepo-2" 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "0" 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG "56555376.16666667" 17:11:03 DEBUG ] 17:11:03 DEBUG ] 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG }, 17:11:03 DEBUG ] 17:11:03 DEBUG ] 17:11:03 DEBUG "0.00002533402181631962" 17:11:03 DEBUG ] 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG "2329.866666666667" 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG ] 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG "value": [ 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG "0" 17:11:03 DEBUG } 17:11:03 DEBUG "0" 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG 1713543063, 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG ] 17:11:03 DEBUG ] 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG "0" 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG ] 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:11:03 DEBUG } 17:12:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543122" 17:12:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543122" 17:12:02 INFO [http_cmd]: http status code OK 17:12:02 DEBUG --- status code --- 17:12:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:02 DEBUG --- http response --- 17:12:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:12:02 INFO [http_cmd]: http status code OK 17:12:02 DEBUG --- status code --- 17:12:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:02 DEBUG --- http response --- 17:12:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:12:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543122" 17:12:02 ERROR [http_cmd]: ERROR 17:12:02 ERROR --- status code --- 17:12:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:12:02 ERROR --- http response --- 17:12:02 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:12:02 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:12:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543122" 17:12:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543122" 17:12:02 INFO [http_cmd]: http status code OK 17:12:02 DEBUG --- status code --- 17:12:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:02 DEBUG --- http response --- 17:12:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:12:02 DEBUG ----- output ----- 17:12:02 DEBUG { 17:12:02 DEBUG "status": "success", 17:12:02 DEBUG "data": { 17:12:02 DEBUG "resultType": "vector", 17:12:02 DEBUG "result": [ 17:12:02 DEBUG { 17:12:02 DEBUG "metric": { 17:12:02 DEBUG "pod": "ds-idrepo-0" 17:12:02 DEBUG }, 17:12:02 DEBUG "value": [ 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG [...] 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG "pod": "ds-cts-2" 17:12:02 DEBUG }, 17:12:02 DEBUG "value": [ 17:12:02 DEBUG 1713543122, 17:12:02 DEBUG "0" 17:12:02 DEBUG ] 17:12:02 DEBUG } 17:12:02 DEBUG ] 17:12:02 DEBUG } 17:12:02 DEBUG } 17:12:02 INFO [http_cmd]: http status code OK 17:12:02 DEBUG --- status code --- 17:12:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:02 DEBUG --- http response --- 17:12:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:12:02 DEBUG ----- output ----- 17:12:02 DEBUG { 17:12:02 DEBUG "status": "success", 17:12:02 DEBUG "data": { 17:12:02 DEBUG "resultType": "vector", 17:12:02 DEBUG "result": [ 17:12:02 DEBUG { 17:12:02 DEBUG "metric": { 17:12:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:12:02 DEBUG }, 17:12:02 DEBUG "value": [ 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG [...] 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:12:02 DEBUG }, 17:12:02 DEBUG "value": [ 17:12:02 DEBUG 1713543122, 17:12:02 DEBUG "2339.4333333333334" 17:12:02 DEBUG ] 17:12:02 DEBUG } 17:12:02 DEBUG ] 17:12:02 DEBUG } 17:12:02 DEBUG } 17:12:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543122" 17:12:02 INFO [http_cmd]: http status code OK 17:12:02 DEBUG --- status code --- 17:12:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:02 DEBUG --- http response --- 17:12:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:12:02 DEBUG ----- output ----- 17:12:02 DEBUG { 17:12:02 DEBUG "status": "success", 17:12:02 DEBUG "data": { 17:12:02 DEBUG "resultType": "vector", 17:12:02 DEBUG "result": [ 17:12:02 DEBUG { 17:12:02 DEBUG "metric": { 17:12:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:12:02 DEBUG }, 17:12:02 DEBUG "value": [ 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG [...] 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:12:02 DEBUG }, 17:12:02 DEBUG "value": [ 17:12:02 DEBUG 1713543122, 17:12:02 DEBUG "0" 17:12:02 DEBUG ] 17:12:02 DEBUG } 17:12:02 DEBUG ] 17:12:02 DEBUG } 17:12:02 DEBUG } 17:12:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543122" 17:12:02 INFO [http_cmd]: http status code OK 17:12:02 DEBUG --- status code --- 17:12:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:02 DEBUG --- http response --- 17:12:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:12:02 DEBUG ----- output ----- 17:12:02 DEBUG { 17:12:02 DEBUG "status": "success", 17:12:02 DEBUG "data": { 17:12:02 DEBUG "resultType": "vector", 17:12:02 DEBUG "result": [ 17:12:02 DEBUG { 17:12:02 DEBUG "metric": { 17:12:02 DEBUG "instance": "10.142.0.107:9100" 17:12:02 DEBUG }, 17:12:02 DEBUG "value": [ 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG [...] 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG "instance": "10.142.0.43:9100" 17:12:02 DEBUG }, 17:12:02 DEBUG "value": [ 17:12:02 DEBUG 1713543122, 17:12:02 DEBUG "0.04000000000000341" 17:12:02 DEBUG ] 17:12:02 DEBUG } 17:12:02 DEBUG ] 17:12:02 DEBUG } 17:12:02 DEBUG } 17:12:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543122" 17:12:02 INFO [http_cmd]: http status code OK 17:12:02 DEBUG --- status code --- 17:12:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:02 DEBUG --- http response --- 17:12:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:12:02 DEBUG ----- output ----- 17:12:02 DEBUG { 17:12:02 DEBUG "status": "success", 17:12:02 DEBUG "data": { 17:12:02 DEBUG "resultType": "vector", 17:12:02 DEBUG "result": [ 17:12:02 DEBUG { 17:12:02 DEBUG "metric": { 17:12:02 DEBUG "pod": "ds-idrepo-1" 17:12:02 DEBUG }, 17:12:02 DEBUG "value": [ 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG [...] 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG 17:12:02 DEBUG "pod": "ds-idrepo-2" 17:12:02 DEBUG }, 17:12:02 DEBUG "value": [ 17:12:02 DEBUG 1713543122, 17:12:02 DEBUG "155832800.98557505" 17:12:02 DEBUG ] 17:12:02 DEBUG } 17:12:02 DEBUG ] 17:12:02 DEBUG } 17:12:02 DEBUG } 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG { 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "data": { 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "result": [ 17:12:03 DEBUG { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:12:03 DEBUG }, 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG [...] 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:12:03 DEBUG }, 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG "1647049.5" 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 INFO 17:12:03 INFO [loop_until]: kubectl --namespace=xlou top node 17:12:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:12:03 INFO [loop_until]: OK (rc = 0) 17:12:03 DEBUG --- stdout --- 17:12:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6180Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5411m 18% 6985Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 50m 0% 2675Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11123m 37% 11964Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 8730m 29% 15446Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 135m 0% 5234Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11365m 38% 11630Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12378m 41% 2960Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12953m 43% 10977Mi 9% gke-xlou-cdm-ds-0f601751-j488 6247m 20% 6633Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 8748m 29% 17168Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6327m 21% 7178Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9100m 30% 14885Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7661m 48% 4631Mi 7% 17:12:03 DEBUG --- stderr --- 17:12:03 DEBUG 17:12:03 INFO 17:12:03 INFO [loop_until]: kubectl --namespace=xlou top pods 17:12:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543123" 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 INFO [loop_until]: OK (rc = 0) 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 DEBUG --- status code --- 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG --- stdout --- 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG --- status code --- 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 10975m 8692Mi am-6fd554f8f7-btd4s 11084m 8758Mi am-6fd554f8f7-wxcsj 11936m 8765Mi ds-cts-0 8716m 12399Mi ds-cts-1 8965m 12333Mi ds-cts-2 9356m 12391Mi ds-idrepo-0 5540m 3421Mi ds-idrepo-1 5359m 3357Mi ds-idrepo-2 5562m 3412Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7939m 832Mi 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG --- status code --- 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG --- stderr --- 17:12:03 DEBUG --- http response --- 17:12:03 INFO [http_cmd]: http status code OK 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:12:03 DEBUG --- status code --- 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:12:03 DEBUG 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG { 17:12:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG { 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:12:03 DEBUG --- http response --- 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG "data": { 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "data": { 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "data": { 17:12:03 DEBUG "data": { 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG { 17:12:03 DEBUG ----- output ----- 17:12:03 DEBUG "data": { 17:12:03 DEBUG "data": { 17:12:03 DEBUG "data": { 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "data": { 17:12:03 DEBUG "data": { 17:12:03 DEBUG "data": { 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "data": { 17:12:03 DEBUG "data": { 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG { 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "data": { 17:12:03 DEBUG "data": { 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG { 17:12:03 DEBUG "data": { 17:12:03 DEBUG "status": "success", 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "result": [ 17:12:03 DEBUG { 17:12:03 DEBUG "data": { 17:12:03 DEBUG "data": { 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "result": [ 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "data": { 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "result": [ 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG "pod": "ds-cts-0" 17:12:03 DEBUG }, 17:12:03 DEBUG "resultType": "vector", 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "result": [ 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:12:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "result": [ 17:12:03 DEBUG "instance": "10.142.0.107:9100" 17:12:03 DEBUG "instance": "10.142.0.107:9100" 17:12:03 DEBUG "pod": "ds-idrepo-1" 17:12:03 DEBUG }, 17:12:03 DEBUG { 17:12:03 DEBUG { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "instance": "10.142.0.107:9100" 17:12:03 DEBUG "instance": "10.142.0.107:9100" 17:12:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG "pod": "ds-cts-0" 17:12:03 DEBUG "pod": "ds-idrepo-1" 17:12:03 DEBUG { 17:12:03 DEBUG 17:12:03 DEBUG { 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "pod": "ds-idrepo-0" 17:12:03 DEBUG "pod": "ds-idrepo-1" 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG "metric": { 17:12:03 DEBUG 17:12:03 DEBUG "metric": { 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 17:12:03 DEBUG "pod": "ds-cts-0" 17:12:03 DEBUG "pod": "ds-idrepo-0" 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:12:03 DEBUG 17:12:03 DEBUG "pod": "ds-idrepo-0" 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG }, 17:12:03 DEBUG [...] 17:12:03 DEBUG }, 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG [...] 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG [...] 17:12:03 DEBUG [...] 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG [...] 17:12:03 DEBUG [...] 17:12:03 DEBUG [...] 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG [...] 17:12:03 DEBUG [...] 17:12:03 DEBUG [...] 17:12:03 DEBUG [...] 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG [...] 17:12:03 DEBUG [...] 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG "pod": "ds-cts-2" 17:12:03 DEBUG 17:12:03 DEBUG [...] 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG [...] 17:12:03 DEBUG [...] 17:12:03 DEBUG [...] 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG [...] 17:12:03 DEBUG }, 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:12:03 DEBUG }, 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG "instance": "10.142.0.43:9100" 17:12:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:12:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG "instance": "10.142.0.43:9100" 17:12:03 DEBUG "pod": "ds-idrepo-2" 17:12:03 DEBUG 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG 17:12:03 DEBUG "instance": "10.142.0.43:9100" 17:12:03 DEBUG 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG "pod": "ds-cts-2" 17:12:03 DEBUG "pod": "ds-idrepo-2" 17:12:03 DEBUG 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG "instance": "10.142.0.43:9100" 17:12:03 DEBUG 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG 17:12:03 DEBUG "pod": "ds-idrepo-2" 17:12:03 DEBUG "pod": "ds-idrepo-2" 17:12:03 DEBUG }, 17:12:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG 17:12:03 DEBUG "0" 17:12:03 DEBUG }, 17:12:03 DEBUG "pod": "ds-idrepo-2" 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 17:12:03 DEBUG "968802304" 17:12:03 DEBUG "pod": "ds-idrepo-2" 17:12:03 DEBUG }, 17:12:03 DEBUG }, 17:12:03 DEBUG "value": [ 17:12:03 DEBUG }, 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "7.880352183499238" 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:12:03 DEBUG ] 17:12:03 DEBUG "value": [ 17:12:03 DEBUG }, 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG "pod": "ds-cts-2" 17:12:03 DEBUG ] 17:12:03 DEBUG }, 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "0.059300000000020725" 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG ] 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG }, 17:12:03 DEBUG } 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG "value": [ 17:12:03 DEBUG "1658278140928" 17:12:03 DEBUG "0" 17:12:03 DEBUG }, 17:12:03 DEBUG } 17:12:03 DEBUG "value": [ 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG "56806057.56666666" 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG ] 17:12:03 DEBUG "2339.8" 17:12:03 DEBUG } 17:12:03 DEBUG "0" 17:12:03 DEBUG "2567.26081871345" 17:12:03 DEBUG "value": [ 17:12:03 DEBUG ] 17:12:03 DEBUG "56799760.733333334" 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG "value": [ 17:12:03 DEBUG ] 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG "0" 17:12:03 DEBUG "3.126230675554123" 17:12:03 DEBUG ] 17:12:03 DEBUG "0.0135" 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG "0" 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG 1713543123, 17:12:03 DEBUG } 17:12:03 DEBUG "0" 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG "2339.7666666666664" 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG "0" 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG ] 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:12:03 DEBUG } 17:13:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543182" 17:13:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543182" 17:13:02 INFO [http_cmd]: http status code OK 17:13:02 DEBUG --- status code --- 17:13:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:02 DEBUG --- http response --- 17:13:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:13:02 INFO [http_cmd]: http status code OK 17:13:02 DEBUG --- status code --- 17:13:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:02 DEBUG --- http response --- 17:13:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:13:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543182" 17:13:02 ERROR [http_cmd]: ERROR 17:13:02 ERROR --- status code --- 17:13:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:13:02 ERROR --- http response --- 17:13:02 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:13:02 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:13:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543182" 17:13:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543182" 17:13:02 INFO [http_cmd]: http status code OK 17:13:02 INFO [http_cmd]: http status code OK 17:13:02 DEBUG --- status code --- 17:13:02 DEBUG --- status code --- 17:13:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:02 DEBUG --- http response --- 17:13:02 DEBUG --- http response --- 17:13:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:13:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:13:02 DEBUG ----- output ----- 17:13:02 DEBUG ----- output ----- 17:13:02 DEBUG { 17:13:02 DEBUG { 17:13:02 DEBUG "status": "success", 17:13:02 DEBUG "status": "success", 17:13:02 DEBUG "data": { 17:13:02 DEBUG "data": { 17:13:02 DEBUG "resultType": "vector", 17:13:02 DEBUG "resultType": "vector", 17:13:02 DEBUG "result": [ 17:13:02 DEBUG "result": [ 17:13:02 DEBUG { 17:13:02 DEBUG { 17:13:02 DEBUG "metric": { 17:13:02 DEBUG "metric": { 17:13:02 DEBUG "pod": "ds-idrepo-0" 17:13:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:13:02 DEBUG }, 17:13:02 DEBUG }, 17:13:02 DEBUG "value": [ 17:13:02 DEBUG "value": [ 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG [...] 17:13:02 DEBUG [...] 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG "pod": "ds-cts-2" 17:13:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:13:02 DEBUG }, 17:13:02 DEBUG }, 17:13:02 DEBUG "value": [ 17:13:02 DEBUG "value": [ 17:13:02 DEBUG 1713543182, 17:13:02 DEBUG 1713543182, 17:13:02 DEBUG "0" 17:13:02 DEBUG "2340.3333333333335" 17:13:02 DEBUG ] 17:13:02 DEBUG ] 17:13:02 DEBUG } 17:13:02 DEBUG } 17:13:02 DEBUG ] 17:13:02 DEBUG ] 17:13:02 DEBUG } 17:13:02 DEBUG } 17:13:02 DEBUG } 17:13:02 DEBUG } 17:13:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543182" 17:13:02 INFO [http_cmd]: http status code OK 17:13:02 DEBUG --- status code --- 17:13:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:02 DEBUG --- http response --- 17:13:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:13:02 DEBUG ----- output ----- 17:13:02 DEBUG { 17:13:02 DEBUG "status": "success", 17:13:02 DEBUG "data": { 17:13:02 DEBUG "resultType": "vector", 17:13:02 DEBUG "result": [ 17:13:02 DEBUG { 17:13:02 DEBUG "metric": { 17:13:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:13:02 DEBUG }, 17:13:02 DEBUG "value": [ 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG [...] 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:13:02 DEBUG }, 17:13:02 DEBUG "value": [ 17:13:02 DEBUG 1713543182, 17:13:02 DEBUG "0" 17:13:02 DEBUG ] 17:13:02 DEBUG } 17:13:02 DEBUG ] 17:13:02 DEBUG } 17:13:02 DEBUG } 17:13:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543182" 17:13:02 INFO [http_cmd]: http status code OK 17:13:02 DEBUG --- status code --- 17:13:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:02 DEBUG --- http response --- 17:13:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:13:02 DEBUG ----- output ----- 17:13:02 DEBUG { 17:13:02 DEBUG "status": "success", 17:13:02 DEBUG "data": { 17:13:02 DEBUG "resultType": "vector", 17:13:02 DEBUG "result": [ 17:13:02 DEBUG { 17:13:02 DEBUG "metric": { 17:13:02 DEBUG "instance": "10.142.0.107:9100" 17:13:02 DEBUG }, 17:13:02 DEBUG "value": [ 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG [...] 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG "instance": "10.142.0.43:9100" 17:13:02 DEBUG }, 17:13:02 DEBUG "value": [ 17:13:02 DEBUG 1713543182, 17:13:02 DEBUG "0.03866666666666372" 17:13:02 DEBUG ] 17:13:02 DEBUG } 17:13:02 DEBUG ] 17:13:02 DEBUG } 17:13:02 DEBUG } 17:13:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543182" 17:13:02 INFO [http_cmd]: http status code OK 17:13:02 DEBUG --- status code --- 17:13:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:02 DEBUG --- http response --- 17:13:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:13:02 DEBUG ----- output ----- 17:13:02 DEBUG { 17:13:02 DEBUG "status": "success", 17:13:02 DEBUG "data": { 17:13:02 DEBUG "resultType": "vector", 17:13:02 DEBUG "result": [ 17:13:02 DEBUG { 17:13:02 DEBUG "metric": { 17:13:02 DEBUG "pod": "ds-idrepo-0" 17:13:02 DEBUG }, 17:13:02 DEBUG "value": [ 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG [...] 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG 17:13:02 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:13:02 DEBUG }, 17:13:02 DEBUG "value": [ 17:13:02 DEBUG 1713543182, 17:13:02 DEBUG "0" 17:13:02 DEBUG ] 17:13:02 DEBUG } 17:13:02 DEBUG ] 17:13:02 DEBUG } 17:13:02 DEBUG } 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG { 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "data": { 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "result": [ 17:13:03 DEBUG { 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:13:03 DEBUG }, 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG [...] 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:13:03 DEBUG }, 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG "1787405" 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 INFO 17:13:03 INFO [loop_until]: kubectl --namespace=xlou top node 17:13:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:13:03 INFO [loop_until]: OK (rc = 0) 17:13:03 DEBUG --- stdout --- 17:13:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6180Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5438m 18% 6985Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 46m 0% 2673Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11134m 37% 11955Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 8748m 29% 15452Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5230Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11258m 37% 11626Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12357m 41% 2957Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12948m 43% 10979Mi 9% gke-xlou-cdm-ds-0f601751-j488 6260m 20% 6627Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9095m 30% 17162Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6346m 21% 7163Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9096m 30% 14890Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8068m 50% 4630Mi 7% 17:13:03 DEBUG --- stderr --- 17:13:03 DEBUG 17:13:03 INFO 17:13:03 INFO [loop_until]: kubectl --namespace=xlou top pods 17:13:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543183" 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 INFO [loop_until]: OK (rc = 0) 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 DEBUG --- status code --- 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 DEBUG --- status code --- 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG --- status code --- 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 DEBUG --- stdout --- 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11243m 8693Mi am-6fd554f8f7-btd4s 11179m 8759Mi am-6fd554f8f7-wxcsj 11566m 8767Mi ds-cts-0 8652m 12406Mi ds-cts-1 8795m 12340Mi ds-cts-2 9247m 12398Mi ds-idrepo-0 5767m 3414Mi ds-idrepo-1 5436m 3357Mi ds-idrepo-2 5499m 3414Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7822m 835Mi 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG --- stderr --- 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG --- http response --- 17:13:03 INFO [http_cmd]: http status code OK 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:13:03 DEBUG --- status code --- 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG { 17:13:03 DEBUG { 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG { 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG { 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG { 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG { 17:13:03 DEBUG { 17:13:03 DEBUG { 17:13:03 DEBUG { 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG { 17:13:03 DEBUG { 17:13:03 DEBUG --- http response --- 17:13:03 DEBUG { 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG { 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "data": { 17:13:03 DEBUG "data": { 17:13:03 DEBUG "data": { 17:13:03 DEBUG { 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "data": { 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "data": { 17:13:03 DEBUG { 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "data": { 17:13:03 DEBUG "data": { 17:13:03 DEBUG "data": { 17:13:03 DEBUG "data": { 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG "data": { 17:13:03 DEBUG "data": { 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG "data": { 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "data": { 17:13:03 DEBUG "data": { 17:13:03 DEBUG "result": [ 17:13:03 DEBUG "result": [ 17:13:03 DEBUG "result": [ 17:13:03 DEBUG "data": { 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "result": [ 17:13:03 DEBUG { 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG { 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "result": [ 17:13:03 DEBUG ----- output ----- 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG { 17:13:03 DEBUG { 17:13:03 DEBUG { 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "result": [ 17:13:03 DEBUG "result": [ 17:13:03 DEBUG "result": [ 17:13:03 DEBUG "result": [ 17:13:03 DEBUG { 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "result": [ 17:13:03 DEBUG "result": [ 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "result": [ 17:13:03 DEBUG { 17:13:03 DEBUG { 17:13:03 DEBUG "status": "success", 17:13:03 DEBUG "result": [ 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "result": [ 17:13:03 DEBUG { 17:13:03 DEBUG { 17:13:03 DEBUG { 17:13:03 DEBUG { 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "data": { 17:13:03 DEBUG { 17:13:03 DEBUG { 17:13:03 DEBUG "metric": { 17:13:03 DEBUG { 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "data": { 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "result": [ 17:13:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:13:03 DEBUG "instance": "10.142.0.107:9100" 17:13:03 DEBUG "instance": "10.142.0.107:9100" 17:13:03 DEBUG { 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "instance": "10.142.0.107:9100" 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "pod": "ds-idrepo-0" 17:13:03 DEBUG "data": { 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:13:03 DEBUG { 17:13:03 DEBUG "result": [ 17:13:03 DEBUG { 17:13:03 DEBUG "metric": { 17:13:03 DEBUG }, 17:13:03 DEBUG }, 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "metric": { 17:13:03 DEBUG }, 17:13:03 DEBUG "pod": "ds-cts-0" 17:13:03 DEBUG "pod": "ds-idrepo-0" 17:13:03 DEBUG }, 17:13:03 DEBUG "result": [ 17:13:03 DEBUG "pod": "ds-idrepo-0" 17:13:03 DEBUG }, 17:13:03 DEBUG "resultType": "vector", 17:13:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:13:03 DEBUG }, 17:13:03 DEBUG "metric": { 17:13:03 DEBUG { 17:13:03 DEBUG "pod": "ds-cts-0" 17:13:03 DEBUG "value": [ 17:13:03 DEBUG "value": [ 17:13:03 DEBUG "pod": "ds-cts-0" 17:13:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:13:03 DEBUG "value": [ 17:13:03 DEBUG }, 17:13:03 DEBUG }, 17:13:03 DEBUG }, 17:13:03 DEBUG "value": [ 17:13:03 DEBUG { 17:13:03 DEBUG }, 17:13:03 DEBUG "value": [ 17:13:03 DEBUG "result": [ 17:13:03 DEBUG }, 17:13:03 DEBUG "value": [ 17:13:03 DEBUG "pod": "ds-idrepo-0" 17:13:03 DEBUG "metric": { 17:13:03 DEBUG }, 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG }, 17:13:03 DEBUG }, 17:13:03 DEBUG 17:13:03 DEBUG "value": [ 17:13:03 DEBUG "value": [ 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 17:13:03 DEBUG "metric": { 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 17:13:03 DEBUG { 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 17:13:03 DEBUG }, 17:13:03 DEBUG "pod": "ds-idrepo-0" 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG "value": [ 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG "instance": "10.142.0.107:9100" 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG "metric": { 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG "value": [ 17:13:03 DEBUG }, 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG }, 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG "pod": "ds-idrepo-0" 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 17:13:03 DEBUG [...] 17:13:03 DEBUG [...] 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG [...] 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG [...] 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 17:13:03 DEBUG [...] 17:13:03 DEBUG }, 17:13:03 DEBUG 17:13:03 DEBUG [...] 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG [...] 17:13:03 DEBUG [...] 17:13:03 DEBUG [...] 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG [...] 17:13:03 DEBUG 17:13:03 DEBUG "value": [ 17:13:03 DEBUG [...] 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG [...] 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG [...] 17:13:03 DEBUG [...] 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG [...] 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG [...] 17:13:03 DEBUG 17:13:03 DEBUG "instance": "10.142.0.43:9100" 17:13:03 DEBUG "instance": "10.142.0.43:9100" 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG "instance": "10.142.0.43:9100" 17:13:03 DEBUG [...] 17:13:03 DEBUG 17:13:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG }, 17:13:03 DEBUG }, 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG }, 17:13:03 DEBUG "pod": "ds-cts-2" 17:13:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:13:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:13:03 DEBUG }, 17:13:03 DEBUG 17:13:03 DEBUG "pod": "ds-idrepo-2" 17:13:03 DEBUG }, 17:13:03 DEBUG [...] 17:13:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:13:03 DEBUG }, 17:13:03 DEBUG 17:13:03 DEBUG 17:13:03 DEBUG "pod": "ds-cts-2" 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG "value": [ 17:13:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:13:03 DEBUG "value": [ 17:13:03 DEBUG }, 17:13:03 DEBUG }, 17:13:03 DEBUG }, 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 17:13:03 DEBUG }, 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 17:13:03 DEBUG }, 17:13:03 DEBUG "value": [ 17:13:03 DEBUG "pod": "ds-idrepo-2" 17:13:03 DEBUG 17:13:03 DEBUG }, 17:13:03 DEBUG "56821122.86666667" 17:13:03 DEBUG "pod": "ds-cts-2" 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG }, 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG "value": [ 17:13:03 DEBUG "value": [ 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG }, 17:13:03 DEBUG "pod": "ds-idrepo-2" 17:13:03 DEBUG "value": [ 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG }, 17:13:03 DEBUG "1660975229952" 17:13:03 DEBUG "value": [ 17:13:03 DEBUG "2340.233333333333" 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG "0.060600000000023135" 17:13:03 DEBUG "instance": "10.142.0.43:9100" 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG "0" 17:13:03 DEBUG 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG "970285056" 17:13:03 DEBUG "value": [ 17:13:03 DEBUG }, 17:13:03 DEBUG "0" 17:13:03 DEBUG "value": [ 17:13:03 DEBUG ] 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG ] 17:13:03 DEBUG "0" 17:13:03 DEBUG ] 17:13:03 DEBUG "2340.3333333333335" 17:13:03 DEBUG "0.000035292035398227074" 17:13:03 DEBUG ] 17:13:03 DEBUG }, 17:13:03 DEBUG "0" 17:13:03 DEBUG ] 17:13:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:13:03 DEBUG "0.014499999999999999" 17:13:03 DEBUG ] 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG "value": [ 17:13:03 DEBUG ] 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG } 17:13:03 DEBUG "7.8930881689409285" 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG "value": [ 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG }, 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG "0" 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG } 17:13:03 DEBUG "0" 17:13:03 DEBUG ] 17:13:03 DEBUG ] 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG "value": [ 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG ] 17:13:03 DEBUG "0" 17:13:03 DEBUG ] 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG "56827357.5" 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG 1713543183, 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG "0" 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG ] 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:13:03 DEBUG } 17:14:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543242" 17:14:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543242" 17:14:02 INFO [http_cmd]: http status code OK 17:14:02 DEBUG --- status code --- 17:14:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:02 DEBUG --- http response --- 17:14:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:14:02 INFO [http_cmd]: http status code OK 17:14:02 DEBUG --- status code --- 17:14:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:02 DEBUG --- http response --- 17:14:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:14:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543242" 17:14:02 ERROR [http_cmd]: ERROR 17:14:02 ERROR --- status code --- 17:14:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:14:02 ERROR --- http response --- 17:14:02 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:14:02 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:14:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543242" 17:14:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543242" 17:14:02 INFO [http_cmd]: http status code OK 17:14:02 DEBUG --- status code --- 17:14:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:02 DEBUG --- http response --- 17:14:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:14:02 DEBUG ----- output ----- 17:14:02 DEBUG { 17:14:02 DEBUG "status": "success", 17:14:02 DEBUG "data": { 17:14:02 DEBUG "resultType": "vector", 17:14:02 DEBUG "result": [ 17:14:02 DEBUG { 17:14:02 DEBUG "metric": { 17:14:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:14:02 DEBUG }, 17:14:02 DEBUG "value": [ 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG [...] 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:14:02 DEBUG }, 17:14:02 DEBUG "value": [ 17:14:02 DEBUG 1713543242, 17:14:02 DEBUG "2326.4" 17:14:02 DEBUG ] 17:14:02 DEBUG } 17:14:02 DEBUG ] 17:14:02 DEBUG } 17:14:02 DEBUG } 17:14:02 INFO [http_cmd]: http status code OK 17:14:02 DEBUG --- status code --- 17:14:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:02 DEBUG --- http response --- 17:14:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:14:02 DEBUG ----- output ----- 17:14:02 DEBUG { 17:14:02 DEBUG "status": "success", 17:14:02 DEBUG "data": { 17:14:02 DEBUG "resultType": "vector", 17:14:02 DEBUG "result": [ 17:14:02 DEBUG { 17:14:02 DEBUG "metric": { 17:14:02 DEBUG "pod": "ds-idrepo-0" 17:14:02 DEBUG }, 17:14:02 DEBUG "value": [ 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG [...] 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG "pod": "ds-cts-2" 17:14:02 DEBUG }, 17:14:02 DEBUG "value": [ 17:14:02 DEBUG 1713543242, 17:14:02 DEBUG "0" 17:14:02 DEBUG ] 17:14:02 DEBUG } 17:14:02 DEBUG ] 17:14:02 DEBUG } 17:14:02 DEBUG } 17:14:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543242" 17:14:02 INFO [http_cmd]: http status code OK 17:14:02 DEBUG --- status code --- 17:14:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:02 DEBUG --- http response --- 17:14:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:14:02 DEBUG ----- output ----- 17:14:02 DEBUG { 17:14:02 DEBUG "status": "success", 17:14:02 DEBUG "data": { 17:14:02 DEBUG "resultType": "vector", 17:14:02 DEBUG "result": [ 17:14:02 DEBUG { 17:14:02 DEBUG "metric": { 17:14:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:14:02 DEBUG }, 17:14:02 DEBUG "value": [ 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG [...] 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:14:02 DEBUG }, 17:14:02 DEBUG "value": [ 17:14:02 DEBUG 1713543242, 17:14:02 DEBUG "0" 17:14:02 DEBUG ] 17:14:02 DEBUG } 17:14:02 DEBUG ] 17:14:02 DEBUG } 17:14:02 DEBUG } 17:14:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543242" 17:14:02 INFO [http_cmd]: http status code OK 17:14:02 DEBUG --- status code --- 17:14:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:02 DEBUG --- http response --- 17:14:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 17:14:02 DEBUG ----- output ----- 17:14:02 DEBUG { 17:14:02 DEBUG "status": "success", 17:14:02 DEBUG "data": { 17:14:02 DEBUG "resultType": "vector", 17:14:02 DEBUG "result": [ 17:14:02 DEBUG { 17:14:02 DEBUG "metric": { 17:14:02 DEBUG "pod": "ds-idrepo-0" 17:14:02 DEBUG }, 17:14:02 DEBUG "value": [ 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG [...] 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:14:02 DEBUG }, 17:14:02 DEBUG "value": [ 17:14:02 DEBUG 1713543242, 17:14:02 DEBUG "0" 17:14:02 DEBUG ] 17:14:02 DEBUG } 17:14:02 DEBUG ] 17:14:02 DEBUG } 17:14:02 DEBUG } 17:14:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543242" 17:14:02 INFO [http_cmd]: http status code OK 17:14:02 DEBUG --- status code --- 17:14:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:02 DEBUG --- http response --- 17:14:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:14:02 DEBUG ----- output ----- 17:14:02 DEBUG { 17:14:02 DEBUG "status": "success", 17:14:02 DEBUG "data": { 17:14:02 DEBUG "resultType": "vector", 17:14:02 DEBUG "result": [ 17:14:02 DEBUG { 17:14:02 DEBUG "metric": { 17:14:02 DEBUG "instance": "10.142.0.107:9100" 17:14:02 DEBUG }, 17:14:02 DEBUG "value": [ 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG [...] 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG 17:14:02 DEBUG "instance": "10.142.0.43:9100" 17:14:02 DEBUG }, 17:14:02 DEBUG "value": [ 17:14:02 DEBUG 1713543242, 17:14:02 DEBUG "0.04200000000000538" 17:14:02 DEBUG ] 17:14:02 DEBUG } 17:14:02 DEBUG ] 17:14:02 DEBUG } 17:14:02 DEBUG } 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG { 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "data": { 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "result": [ 17:14:03 DEBUG { 17:14:03 DEBUG "metric": { 17:14:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:14:03 DEBUG }, 17:14:03 DEBUG "value": [ 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG [...] 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:14:03 DEBUG }, 17:14:03 DEBUG "value": [ 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG "1927412.5" 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 INFO 17:14:03 INFO [loop_until]: kubectl --namespace=xlou top node 17:14:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:14:03 INFO [loop_until]: OK (rc = 0) 17:14:03 DEBUG --- stdout --- 17:14:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6180Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5360m 17% 6983Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 46m 0% 2680Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 10868m 36% 11969Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 8873m 29% 15452Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5220Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11308m 37% 11626Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12196m 40% 2971Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12906m 43% 10971Mi 9% gke-xlou-cdm-ds-0f601751-j488 6212m 20% 6638Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9082m 30% 17172Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6274m 21% 7167Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9111m 30% 14890Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8012m 50% 4631Mi 7% 17:14:03 DEBUG --- stderr --- 17:14:03 DEBUG 17:14:03 INFO 17:14:03 INFO [loop_until]: kubectl --namespace=xlou top pods 17:14:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543243" 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG { 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "data": { 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "result": [ 17:14:03 DEBUG { 17:14:03 DEBUG "metric": { 17:14:03 DEBUG "instance": "10.142.0.107:9100" 17:14:03 DEBUG }, 17:14:03 DEBUG "value": [ 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG [...] 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG "instance": "10.142.0.43:9100" 17:14:03 DEBUG }, 17:14:03 DEBUG "value": [ 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG "1663697462272" 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543243" 17:14:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543243" 17:14:03 INFO [loop_until]: OK (rc = 0) 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 DEBUG --- stdout --- 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG --- status code --- 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 DEBUG --- status code --- 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 10974m 8694Mi am-6fd554f8f7-btd4s 11189m 8761Mi am-6fd554f8f7-wxcsj 12370m 8770Mi ds-cts-0 8818m 12414Mi ds-cts-1 8975m 12347Mi ds-cts-2 9227m 12406Mi ds-idrepo-0 5602m 3414Mi ds-idrepo-1 5412m 3357Mi ds-idrepo-2 5506m 3414Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7718m 836Mi 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG --- status code --- 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 INFO [http_cmd]: http status code OK 17:14:03 DEBUG --- stderr --- 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG --- status code --- 17:14:03 DEBUG 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:14:03 DEBUG --- http response --- 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG { 17:14:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG { 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG ----- output ----- 17:14:03 DEBUG "data": { 17:14:03 DEBUG "data": { 17:14:03 DEBUG "data": { 17:14:03 DEBUG "data": { 17:14:03 DEBUG "data": { 17:14:03 DEBUG "data": { 17:14:03 DEBUG "data": { 17:14:03 DEBUG "data": { 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "data": { 17:14:03 DEBUG "data": { 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG { 17:14:03 DEBUG "data": { 17:14:03 DEBUG "data": { 17:14:03 DEBUG { 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "data": { 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "data": { 17:14:03 DEBUG "data": { 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "status": "success", 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "data": { 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "data": { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG "metric": { 17:14:03 DEBUG "result": [ 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG "resultType": "vector", 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "metric": { 17:14:03 DEBUG "metric": { 17:14:03 DEBUG "instance": "10.142.0.107:9100" 17:14:03 DEBUG "metric": { 17:14:03 DEBUG "metric": { 17:14:03 DEBUG { 17:14:03 DEBUG "pod": "ds-cts-0" 17:14:03 DEBUG { 17:14:03 DEBUG "metric": { 17:14:03 DEBUG "metric": { 17:14:03 DEBUG { 17:14:03 DEBUG { 17:14:03 DEBUG "result": [ 17:14:03 DEBUG "metric": { 17:14:03 DEBUG "metric": { 17:14:03 DEBUG "metric": { 17:14:03 DEBUG { 17:14:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:14:03 DEBUG "metric": { 17:14:03 DEBUG }, 17:14:03 DEBUG "pod": "ds-cts-0" 17:14:03 DEBUG "pod": "ds-idrepo-0" 17:14:03 DEBUG "metric": { 17:14:03 DEBUG }, 17:14:03 DEBUG "metric": { 17:14:03 DEBUG "pod": "ds-idrepo-0" 17:14:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:14:03 DEBUG "metric": { 17:14:03 DEBUG "metric": { 17:14:03 DEBUG { 17:14:03 DEBUG "instance": "10.142.0.107:9100" 17:14:03 DEBUG "pod": "ds-idrepo-0" 17:14:03 DEBUG "pod": "ds-idrepo-0" 17:14:03 DEBUG "metric": { 17:14:03 DEBUG }, 17:14:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:14:03 DEBUG "value": [ 17:14:03 DEBUG }, 17:14:03 DEBUG }, 17:14:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "pod": "ds-cts-0" 17:14:03 DEBUG }, 17:14:03 DEBUG }, 17:14:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:14:03 DEBUG "pod": "ds-idrepo-0" 17:14:03 DEBUG "metric": { 17:14:03 DEBUG }, 17:14:03 DEBUG }, 17:14:03 DEBUG }, 17:14:03 DEBUG "instance": "10.142.0.107:9100" 17:14:03 DEBUG "value": [ 17:14:03 DEBUG }, 17:14:03 DEBUG 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "value": [ 17:14:03 DEBUG }, 17:14:03 DEBUG 17:14:03 DEBUG }, 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "value": [ 17:14:03 DEBUG }, 17:14:03 DEBUG }, 17:14:03 DEBUG "pod": "ds-idrepo-0" 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "value": [ 17:14:03 DEBUG }, 17:14:03 DEBUG 17:14:03 DEBUG "value": [ 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG "value": [ 17:14:03 DEBUG 17:14:03 DEBUG "value": [ 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "value": [ 17:14:03 DEBUG }, 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG "value": [ 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG "value": [ 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG [...] 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG [...] 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG [...] 17:14:03 DEBUG [...] 17:14:03 DEBUG [...] 17:14:03 DEBUG [...] 17:14:03 DEBUG 17:14:03 DEBUG [...] 17:14:03 DEBUG [...] 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG [...] 17:14:03 DEBUG [...] 17:14:03 DEBUG [...] 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG [...] 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG [...] 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG [...] 17:14:03 DEBUG [...] 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG [...] 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG [...] 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG "instance": "10.142.0.43:9100" 17:14:03 DEBUG 17:14:03 DEBUG "pod": "ds-cts-2" 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:14:03 DEBUG 17:14:03 DEBUG }, 17:14:03 DEBUG 17:14:03 DEBUG }, 17:14:03 DEBUG "pod": "ds-idrepo-2" 17:14:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:14:03 DEBUG 17:14:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:14:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG 17:14:03 DEBUG "instance": "10.142.0.43:9100" 17:14:03 DEBUG "pod": "ds-idrepo-2" 17:14:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:14:03 DEBUG 17:14:03 DEBUG }, 17:14:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "pod": "ds-cts-2" 17:14:03 DEBUG "value": [ 17:14:03 DEBUG }, 17:14:03 DEBUG }, 17:14:03 DEBUG "pod": "ds-cts-2" 17:14:03 DEBUG }, 17:14:03 DEBUG }, 17:14:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:14:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:14:03 DEBUG 17:14:03 DEBUG }, 17:14:03 DEBUG }, 17:14:03 DEBUG }, 17:14:03 DEBUG "instance": "10.142.0.43:9100" 17:14:03 DEBUG "value": [ 17:14:03 DEBUG }, 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG }, 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "value": [ 17:14:03 DEBUG }, 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "value": [ 17:14:03 DEBUG }, 17:14:03 DEBUG }, 17:14:03 DEBUG "pod": "ds-idrepo-2" 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "value": [ 17:14:03 DEBUG }, 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "0.06276666666663952" 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "0.1" 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG "value": [ 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "value": [ 17:14:03 DEBUG }, 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "2326.5333333333333" 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG ] 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG ] 17:14:03 DEBUG "0" 17:14:03 DEBUG "2326.5333333333333" 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG "0.00003052143606401917" 17:14:03 DEBUG "0.015" 17:14:03 DEBUG ] 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG "value": [ 17:14:03 DEBUG "56429370.63333333" 17:14:03 DEBUG "0" 17:14:03 DEBUG "0" 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG ] 17:14:03 DEBUG "971669504" 17:14:03 DEBUG } 17:14:03 DEBUG "0" 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG "0" 17:14:03 DEBUG ] 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG } 17:14:03 DEBUG "7.880052276517185" 17:14:03 DEBUG 1713543243, 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG "56435874.93333333" 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG "0" 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG "0" 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG ] 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:14:03 DEBUG } 17:15:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543302" 17:15:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543302" 17:15:02 INFO [http_cmd]: http status code OK 17:15:02 INFO [http_cmd]: http status code OK 17:15:02 DEBUG --- status code --- 17:15:02 DEBUG --- status code --- 17:15:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:02 DEBUG --- http response --- 17:15:02 DEBUG --- http response --- 17:15:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:15:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:15:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543302" 17:15:02 ERROR [http_cmd]: ERROR 17:15:02 ERROR --- status code --- 17:15:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:15:02 ERROR --- http response --- 17:15:02 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:15:02 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:15:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543302" 17:15:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543302" 17:15:02 INFO [http_cmd]: http status code OK 17:15:02 DEBUG --- status code --- 17:15:02 INFO [http_cmd]: http status code OK 17:15:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:02 DEBUG --- status code --- 17:15:02 DEBUG --- http response --- 17:15:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:15:02 DEBUG --- http response --- 17:15:02 DEBUG ----- output ----- 17:15:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:15:02 DEBUG { 17:15:02 DEBUG ----- output ----- 17:15:02 DEBUG "status": "success", 17:15:02 DEBUG { 17:15:02 DEBUG "data": { 17:15:02 DEBUG "status": "success", 17:15:02 DEBUG "resultType": "vector", 17:15:02 DEBUG "data": { 17:15:02 DEBUG "result": [ 17:15:02 DEBUG "resultType": "vector", 17:15:02 DEBUG { 17:15:02 DEBUG "result": [ 17:15:02 DEBUG "metric": { 17:15:02 DEBUG { 17:15:02 DEBUG "pod": "ds-idrepo-0" 17:15:02 DEBUG "metric": { 17:15:02 DEBUG }, 17:15:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:15:02 DEBUG "value": [ 17:15:02 DEBUG }, 17:15:02 DEBUG 17:15:02 DEBUG "value": [ 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG [...] 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG [...] 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG "pod": "ds-cts-2" 17:15:02 DEBUG 17:15:02 DEBUG }, 17:15:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:15:02 DEBUG "value": [ 17:15:02 DEBUG }, 17:15:02 DEBUG 1713543302, 17:15:02 DEBUG "value": [ 17:15:02 DEBUG "0" 17:15:02 DEBUG 1713543302, 17:15:02 DEBUG ] 17:15:02 DEBUG "2344.3333333333335" 17:15:02 DEBUG } 17:15:02 DEBUG ] 17:15:02 DEBUG ] 17:15:02 DEBUG } 17:15:02 DEBUG } 17:15:02 DEBUG ] 17:15:02 DEBUG } 17:15:02 DEBUG } 17:15:02 DEBUG } 17:15:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543302" 17:15:02 INFO [http_cmd]: http status code OK 17:15:02 DEBUG --- status code --- 17:15:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:02 DEBUG --- http response --- 17:15:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:15:02 DEBUG ----- output ----- 17:15:02 DEBUG { 17:15:02 DEBUG "status": "success", 17:15:02 DEBUG "data": { 17:15:02 DEBUG "resultType": "vector", 17:15:02 DEBUG "result": [ 17:15:02 DEBUG { 17:15:02 DEBUG "metric": { 17:15:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:15:02 DEBUG }, 17:15:02 DEBUG "value": [ 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG [...] 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:15:02 DEBUG }, 17:15:02 DEBUG "value": [ 17:15:02 DEBUG 1713543302, 17:15:02 DEBUG "0" 17:15:02 DEBUG ] 17:15:02 DEBUG } 17:15:02 DEBUG ] 17:15:02 DEBUG } 17:15:02 DEBUG } 17:15:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543302" 17:15:02 INFO [http_cmd]: http status code OK 17:15:02 DEBUG --- status code --- 17:15:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:02 DEBUG --- http response --- 17:15:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:15:02 DEBUG ----- output ----- 17:15:02 DEBUG { 17:15:02 DEBUG "status": "success", 17:15:02 DEBUG "data": { 17:15:02 DEBUG "resultType": "vector", 17:15:02 DEBUG "result": [ 17:15:02 DEBUG { 17:15:02 DEBUG "metric": { 17:15:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:15:02 DEBUG }, 17:15:02 DEBUG "value": [ 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG [...] 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG "pod": "ds-idrepo-2" 17:15:02 DEBUG }, 17:15:02 DEBUG "value": [ 17:15:02 DEBUG 1713543302, 17:15:02 DEBUG "0" 17:15:02 DEBUG ] 17:15:02 DEBUG } 17:15:02 DEBUG ] 17:15:02 DEBUG } 17:15:02 DEBUG } 17:15:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543302" 17:15:02 INFO [http_cmd]: http status code OK 17:15:02 DEBUG --- status code --- 17:15:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:02 DEBUG --- http response --- 17:15:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:15:02 DEBUG ----- output ----- 17:15:02 DEBUG { 17:15:02 DEBUG "status": "success", 17:15:02 DEBUG "data": { 17:15:02 DEBUG "resultType": "vector", 17:15:02 DEBUG "result": [ 17:15:02 DEBUG { 17:15:02 DEBUG "metric": { 17:15:02 DEBUG "instance": "10.142.0.107:9100" 17:15:02 DEBUG }, 17:15:02 DEBUG "value": [ 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG [...] 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG 17:15:02 DEBUG "instance": "10.142.0.43:9100" 17:15:02 DEBUG }, 17:15:02 DEBUG "value": [ 17:15:02 DEBUG 1713543302, 17:15:02 DEBUG "0.03866666666667318" 17:15:02 DEBUG ] 17:15:02 DEBUG } 17:15:02 DEBUG ] 17:15:02 DEBUG } 17:15:02 DEBUG } 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG { 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG "data": { 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "result": [ 17:15:03 DEBUG { 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:15:03 DEBUG }, 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG [...] 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:15:03 DEBUG }, 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG "2067433.5" 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 INFO 17:15:03 INFO [loop_until]: kubectl --namespace=xlou top node 17:15:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:15:03 INFO [loop_until]: OK (rc = 0) 17:15:03 DEBUG --- stdout --- 17:15:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6183Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5422m 18% 6974Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 46m 0% 2678Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11197m 37% 11984Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9122m 30% 15440Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5222Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11347m 38% 11628Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12548m 42% 2952Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12978m 43% 10972Mi 9% gke-xlou-cdm-ds-0f601751-j488 6282m 21% 6639Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9114m 30% 17164Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6368m 21% 7170Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9460m 31% 14886Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8100m 50% 4630Mi 7% 17:15:03 DEBUG --- stderr --- 17:15:03 DEBUG 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543303" 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG { 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG "data": { 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "result": [ 17:15:03 DEBUG { 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "instance": "10.142.0.107:9100" 17:15:03 DEBUG }, 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG [...] 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG "instance": "10.142.0.43:9100" 17:15:03 DEBUG }, 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG "1666460951552" 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 INFO 17:15:03 INFO [loop_until]: kubectl --namespace=xlou top pods 17:15:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543303" 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 INFO [loop_until]: OK (rc = 0) 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 DEBUG --- status code --- 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 DEBUG --- status code --- 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 DEBUG --- status code --- 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 DEBUG --- stdout --- 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG --- status code --- 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11310m 8704Mi am-6fd554f8f7-btd4s 11422m 8760Mi am-6fd554f8f7-wxcsj 12078m 8770Mi ds-cts-0 9014m 12419Mi ds-cts-1 9491m 12355Mi ds-cts-2 9424m 12412Mi ds-idrepo-0 5737m 3414Mi ds-idrepo-1 5557m 3363Mi ds-idrepo-2 5489m 3420Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7693m 840Mi 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG --- stderr --- 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG { 17:15:03 DEBUG { 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG { 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG { 17:15:03 DEBUG { 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG { 17:15:03 DEBUG { 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG { 17:15:03 DEBUG { 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG { 17:15:03 DEBUG { 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG { 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG { 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG "data": { 17:15:03 DEBUG "data": { 17:15:03 INFO [http_cmd]: http status code OK 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG "data": { 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG { 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG "data": { 17:15:03 DEBUG "data": { 17:15:03 DEBUG "data": { 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:15:03 DEBUG "data": { 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG --- status code --- 17:15:03 DEBUG "data": { 17:15:03 DEBUG "data": { 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "result": [ 17:15:03 DEBUG "data": { 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:15:03 DEBUG "data": { 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG "data": { 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "result": [ 17:15:03 DEBUG "result": [ 17:15:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "data": { 17:15:03 DEBUG { 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "data": { 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "result": [ 17:15:03 DEBUG "result": [ 17:15:03 DEBUG "result": [ 17:15:03 DEBUG { 17:15:03 DEBUG "result": [ 17:15:03 DEBUG { 17:15:03 DEBUG { 17:15:03 DEBUG "metric": { 17:15:03 DEBUG --- http response --- 17:15:03 DEBUG "result": [ 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "result": [ 17:15:03 DEBUG { 17:15:03 DEBUG "result": [ 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "result": [ 17:15:03 DEBUG { 17:15:03 DEBUG { 17:15:03 DEBUG { 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG { 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "result": [ 17:15:03 DEBUG "pod": "ds-cts-0" 17:15:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:15:03 DEBUG { 17:15:03 DEBUG "result": [ 17:15:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:15:03 DEBUG { 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG { 17:15:03 DEBUG "result": [ 17:15:03 DEBUG { 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "data": { 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "instance": "10.142.0.107:9100" 17:15:03 DEBUG { 17:15:03 DEBUG }, 17:15:03 DEBUG ----- output ----- 17:15:03 DEBUG "metric": { 17:15:03 DEBUG { 17:15:03 DEBUG }, 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "data": { 17:15:03 DEBUG "metric": { 17:15:03 DEBUG { 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:15:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:15:03 DEBUG "instance": "10.142.0.107:9100" 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:15:03 DEBUG }, 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "value": [ 17:15:03 DEBUG { 17:15:03 DEBUG "pod": "ds-idrepo-0" 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "value": [ 17:15:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG "pod": "ds-cts-0" 17:15:03 DEBUG "metric": { 17:15:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:15:03 DEBUG }, 17:15:03 DEBUG }, 17:15:03 DEBUG }, 17:15:03 DEBUG "result": [ 17:15:03 DEBUG { 17:15:03 DEBUG "value": [ 17:15:03 DEBUG "pod": "ds-cts-0" 17:15:03 DEBUG 17:15:03 DEBUG "status": "success", 17:15:03 DEBUG }, 17:15:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:15:03 DEBUG 17:15:03 DEBUG }, 17:15:03 DEBUG "result": [ 17:15:03 DEBUG }, 17:15:03 DEBUG "pod": "ds-idrepo-0" 17:15:03 DEBUG }, 17:15:03 DEBUG "value": [ 17:15:03 DEBUG "value": [ 17:15:03 DEBUG "value": [ 17:15:03 DEBUG }, 17:15:03 DEBUG "metric": { 17:15:03 DEBUG 17:15:03 DEBUG }, 17:15:03 DEBUG 17:15:03 DEBUG "data": { 17:15:03 DEBUG "value": [ 17:15:03 DEBUG }, 17:15:03 DEBUG 17:15:03 DEBUG "value": [ 17:15:03 DEBUG { 17:15:03 DEBUG "value": [ 17:15:03 DEBUG }, 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG "value": [ 17:15:03 DEBUG "pod": "ds-idrepo-0" 17:15:03 DEBUG 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 17:15:03 DEBUG "resultType": "vector", 17:15:03 DEBUG 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG "metric": { 17:15:03 DEBUG 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG }, 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG [...] 17:15:03 DEBUG 17:15:03 DEBUG "result": [ 17:15:03 DEBUG 17:15:03 DEBUG [...] 17:15:03 DEBUG 17:15:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG "value": [ 17:15:03 DEBUG [...] 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG { 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG }, 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG [...] 17:15:03 DEBUG [...] 17:15:03 DEBUG [...] 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG "metric": { 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG [...] 17:15:03 DEBUG "value": [ 17:15:03 DEBUG [...] 17:15:03 DEBUG 17:15:03 DEBUG [...] 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG [...] 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG [...] 17:15:03 DEBUG [...] 17:15:03 DEBUG "pod": "ds-cts-2" 17:15:03 DEBUG "instance": "10.142.0.107:9100" 17:15:03 DEBUG [...] 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG [...] 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG }, 17:15:03 DEBUG }, 17:15:03 DEBUG 17:15:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG [...] 17:15:03 DEBUG "instance": "10.142.0.43:9100" 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG "value": [ 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 17:15:03 DEBUG }, 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 17:15:03 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 17:15:03 DEBUG "instance": "10.142.0.43:9100" 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG }, 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG 17:15:03 DEBUG 17:15:03 DEBUG "value": [ 17:15:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:15:03 DEBUG [...] 17:15:03 DEBUG "pod": "ds-cts-2" 17:15:03 DEBUG 17:15:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:15:03 DEBUG }, 17:15:03 DEBUG }, 17:15:03 DEBUG }, 17:15:03 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:15:03 DEBUG 17:15:03 DEBUG "value": [ 17:15:03 DEBUG "pod": "ds-cts-2" 17:15:03 DEBUG "pod": "ds-idrepo-2" 17:15:03 DEBUG "0.1" 17:15:03 DEBUG 17:15:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG }, 17:15:03 DEBUG 17:15:03 DEBUG }, 17:15:03 DEBUG "pod": "ds-idrepo-2" 17:15:03 DEBUG }, 17:15:03 DEBUG "value": [ 17:15:03 DEBUG "value": [ 17:15:03 DEBUG "value": [ 17:15:03 DEBUG }, 17:15:03 DEBUG 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG }, 17:15:03 DEBUG }, 17:15:03 DEBUG ] 17:15:03 DEBUG 17:15:03 DEBUG }, 17:15:03 DEBUG "2343.5333333333333" 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 17:15:03 DEBUG "value": [ 17:15:03 DEBUG }, 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG "value": [ 17:15:03 DEBUG "pod": "ds-idrepo-2" 17:15:03 DEBUG "0.062266666666752714" 17:15:03 DEBUG "value": [ 17:15:03 DEBUG "value": [ 17:15:03 DEBUG } 17:15:03 DEBUG [...] 17:15:03 DEBUG 17:15:03 DEBUG ] 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG "0" 17:15:03 DEBUG "0" 17:15:03 DEBUG "56879140.2" 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG }, 17:15:03 DEBUG ] 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG ] 17:15:03 DEBUG "value": [ 17:15:03 DEBUG 17:15:03 DEBUG } 17:15:03 DEBUG "975773696" 17:15:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:15:03 DEBUG "0.000033333333333333335" 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG "0.015" 17:15:03 DEBUG ] 17:15:03 DEBUG ] 17:15:03 DEBUG ] 17:15:03 DEBUG "0" 17:15:03 DEBUG "value": [ 17:15:03 DEBUG } 17:15:03 DEBUG "0" 17:15:03 DEBUG "0" 17:15:03 DEBUG } 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG 17:15:03 DEBUG ] 17:15:03 DEBUG ] 17:15:03 DEBUG }, 17:15:03 DEBUG ] 17:15:03 DEBUG "0" 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG ] 17:15:03 DEBUG ] 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG "2342.3" 17:15:03 DEBUG "instance": "10.142.0.43:9100" 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG "value": [ 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG ] 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG "0" 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG }, 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG "value": [ 17:15:03 DEBUG } 17:15:03 DEBUG "7.909333099893159" 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG 1713543303, 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG "56885133.2" 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG ] 17:15:03 DEBUG } 17:15:03 DEBUG } 17:15:03 DEBUG } 17:16:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543362" 17:16:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543362" 17:16:02 INFO [http_cmd]: http status code OK 17:16:02 DEBUG --- status code --- 17:16:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:02 DEBUG --- http response --- 17:16:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:16:02 INFO [http_cmd]: http status code OK 17:16:02 DEBUG --- status code --- 17:16:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:02 DEBUG --- http response --- 17:16:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:16:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543362" 17:16:02 ERROR [http_cmd]: ERROR 17:16:02 ERROR --- status code --- 17:16:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:16:02 ERROR --- http response --- 17:16:02 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:16:02 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:16:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543362" 17:16:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543362" 17:16:02 INFO [http_cmd]: http status code OK 17:16:02 INFO [http_cmd]: http status code OK 17:16:02 DEBUG --- status code --- 17:16:02 DEBUG --- status code --- 17:16:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:02 DEBUG --- http response --- 17:16:02 DEBUG --- http response --- 17:16:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:16:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:16:02 DEBUG ----- output ----- 17:16:02 DEBUG ----- output ----- 17:16:02 DEBUG { 17:16:02 DEBUG { 17:16:02 DEBUG "status": "success", 17:16:02 DEBUG "status": "success", 17:16:02 DEBUG "data": { 17:16:02 DEBUG "data": { 17:16:02 DEBUG "resultType": "vector", 17:16:02 DEBUG "resultType": "vector", 17:16:02 DEBUG "result": [ 17:16:02 DEBUG "result": [ 17:16:02 DEBUG { 17:16:02 DEBUG { 17:16:02 DEBUG "metric": { 17:16:02 DEBUG "metric": { 17:16:02 DEBUG "pod": "ds-idrepo-0" 17:16:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:16:02 DEBUG }, 17:16:02 DEBUG }, 17:16:02 DEBUG "value": [ 17:16:02 DEBUG "value": [ 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG [...] 17:16:02 DEBUG [...] 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG "pod": "ds-cts-2" 17:16:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:16:02 DEBUG }, 17:16:02 DEBUG }, 17:16:02 DEBUG "value": [ 17:16:02 DEBUG "value": [ 17:16:02 DEBUG 1713543362, 17:16:02 DEBUG 1713543362, 17:16:02 DEBUG "0" 17:16:02 DEBUG "2327.633333333333" 17:16:02 DEBUG ] 17:16:02 DEBUG ] 17:16:02 DEBUG } 17:16:02 DEBUG } 17:16:02 DEBUG ] 17:16:02 DEBUG ] 17:16:02 DEBUG } 17:16:02 DEBUG } 17:16:02 DEBUG } 17:16:02 DEBUG } 17:16:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543362" 17:16:02 INFO [http_cmd]: http status code OK 17:16:02 DEBUG --- status code --- 17:16:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:02 DEBUG --- http response --- 17:16:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:16:02 DEBUG ----- output ----- 17:16:02 DEBUG { 17:16:02 DEBUG "status": "success", 17:16:02 DEBUG "data": { 17:16:02 DEBUG "resultType": "vector", 17:16:02 DEBUG "result": [ 17:16:02 DEBUG { 17:16:02 DEBUG "metric": { 17:16:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:16:02 DEBUG }, 17:16:02 DEBUG "value": [ 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG [...] 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:16:02 DEBUG }, 17:16:02 DEBUG "value": [ 17:16:02 DEBUG 1713543362, 17:16:02 DEBUG "0" 17:16:02 DEBUG ] 17:16:02 DEBUG } 17:16:02 DEBUG ] 17:16:02 DEBUG } 17:16:02 DEBUG } 17:16:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543362" 17:16:02 INFO [http_cmd]: http status code OK 17:16:02 DEBUG --- status code --- 17:16:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:02 DEBUG --- http response --- 17:16:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 17:16:02 DEBUG ----- output ----- 17:16:02 DEBUG { 17:16:02 DEBUG "status": "success", 17:16:02 DEBUG "data": { 17:16:02 DEBUG "resultType": "vector", 17:16:02 DEBUG "result": [ 17:16:02 DEBUG { 17:16:02 DEBUG "metric": { 17:16:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:16:02 DEBUG }, 17:16:02 DEBUG "value": [ 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG [...] 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG "pod": "idm-b85f9d978-txpwv" 17:16:02 DEBUG }, 17:16:02 DEBUG "value": [ 17:16:02 DEBUG 1713543362, 17:16:02 DEBUG "10489.669058805308" 17:16:02 DEBUG ] 17:16:02 DEBUG } 17:16:02 DEBUG ] 17:16:02 DEBUG } 17:16:02 DEBUG } 17:16:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543362" 17:16:02 INFO [http_cmd]: http status code OK 17:16:02 DEBUG --- status code --- 17:16:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:02 DEBUG --- http response --- 17:16:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:16:02 DEBUG ----- output ----- 17:16:02 DEBUG { 17:16:02 DEBUG "status": "success", 17:16:02 DEBUG "data": { 17:16:02 DEBUG "resultType": "vector", 17:16:02 DEBUG "result": [ 17:16:02 DEBUG { 17:16:02 DEBUG "metric": { 17:16:02 DEBUG "instance": "10.142.0.107:9100" 17:16:02 DEBUG }, 17:16:02 DEBUG "value": [ 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG [...] 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG 17:16:02 DEBUG "instance": "10.142.0.43:9100" 17:16:02 DEBUG }, 17:16:02 DEBUG "value": [ 17:16:02 DEBUG 1713543362, 17:16:02 DEBUG "0.04000000000000909" 17:16:02 DEBUG ] 17:16:02 DEBUG } 17:16:02 DEBUG ] 17:16:02 DEBUG } 17:16:02 DEBUG } 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:16:03 DEBUG ----- output ----- 17:16:03 DEBUG { 17:16:03 DEBUG "status": "success", 17:16:03 DEBUG "data": { 17:16:03 DEBUG "resultType": "vector", 17:16:03 DEBUG "result": [ 17:16:03 DEBUG { 17:16:03 DEBUG "metric": { 17:16:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:16:03 DEBUG }, 17:16:03 DEBUG "value": [ 17:16:03 DEBUG 17:16:03 DEBUG 17:16:03 DEBUG 17:16:03 DEBUG [...] 17:16:03 DEBUG 17:16:03 DEBUG 17:16:03 DEBUG 17:16:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:16:03 DEBUG }, 17:16:03 DEBUG "value": [ 17:16:03 DEBUG 1713543363, 17:16:03 DEBUG "2207217" 17:16:03 DEBUG ] 17:16:03 DEBUG } 17:16:03 DEBUG ] 17:16:03 DEBUG } 17:16:03 DEBUG } 17:16:03 INFO 17:16:03 INFO [loop_until]: kubectl --namespace=xlou top node 17:16:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543363" 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:16:03 DEBUG ----- output ----- 17:16:03 DEBUG { 17:16:03 DEBUG "status": "success", 17:16:03 DEBUG "data": { 17:16:03 DEBUG "resultType": "vector", 17:16:03 DEBUG "result": [ 17:16:03 DEBUG { 17:16:03 DEBUG "metric": { 17:16:03 DEBUG "instance": "10.142.0.107:9100" 17:16:03 DEBUG }, 17:16:03 DEBUG "value": [ 17:16:03 DEBUG 17:16:03 DEBUG 17:16:03 DEBUG 17:16:03 DEBUG [...] 17:16:03 DEBUG 17:16:03 DEBUG 17:16:03 DEBUG 17:16:03 DEBUG "instance": "10.142.0.43:9100" 17:16:03 DEBUG }, 17:16:03 DEBUG "value": [ 17:16:03 DEBUG 1713543363, 17:16:03 DEBUG "1669226046464" 17:16:03 DEBUG ] 17:16:03 DEBUG } 17:16:03 DEBUG ] 17:16:03 DEBUG } 17:16:03 DEBUG } 17:16:03 INFO [loop_until]: OK (rc = 0) 17:16:03 DEBUG --- stdout --- 17:16:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6178Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5419m 18% 6974Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 46m 0% 2678Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11144m 37% 11993Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 8963m 30% 15433Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 148m 0% 5231Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11072m 37% 11630Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12252m 41% 2955Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12914m 43% 10973Mi 9% gke-xlou-cdm-ds-0f601751-j488 6196m 20% 6632Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9335m 31% 17177Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6295m 21% 7170Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9463m 31% 14879Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7768m 48% 4629Mi 7% 17:16:03 DEBUG --- stderr --- 17:16:03 DEBUG 17:16:03 INFO 17:16:03 INFO [loop_until]: kubectl --namespace=xlou top pods 17:16:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543363" 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543363" 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 INFO [loop_until]: OK (rc = 0) 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG --- status code --- 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 DEBUG --- status code --- 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 DEBUG --- status code --- 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 DEBUG --- status code --- 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 DEBUG --- status code --- 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 DEBUG --- stdout --- 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG --- status code --- 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11160m 8715Mi am-6fd554f8f7-btd4s 11476m 8763Mi am-6fd554f8f7-wxcsj 11975m 8771Mi ds-cts-0 8998m 12427Mi ds-cts-1 9422m 12362Mi ds-cts-2 9648m 12417Mi ds-idrepo-0 5649m 3414Mi ds-idrepo-1 5429m 3365Mi ds-idrepo-2 5331m 3424Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7758m 842Mi 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 INFO [http_cmd]: http status code OK 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG --- stderr --- 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG --- status code --- 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG ----- output ----- 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:16:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:16:03 DEBUG ----- output ----- 17:16:03 DEBUG ----- output ----- 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 17:16:03 DEBUG ----- output ----- 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:16:03 DEBUG ----- output ----- 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:16:03 DEBUG ----- output ----- 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:16:03 DEBUG { 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:16:03 DEBUG ----- output ----- 17:16:03 DEBUG --- http response --- 17:16:03 DEBUG { 17:16:03 DEBUG { 17:16:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:16:04 DEBUG ----- output ----- 17:16:04 DEBUG { 17:16:04 DEBUG ----- output ----- 17:16:04 DEBUG { 17:16:04 DEBUG ----- output ----- 17:16:04 DEBUG ----- output ----- 17:16:04 DEBUG ----- output ----- 17:16:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:16:04 DEBUG { 17:16:04 DEBUG ----- output ----- 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG ----- output ----- 17:16:04 DEBUG { 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:16:04 DEBUG ----- output ----- 17:16:04 DEBUG { 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG { 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG { 17:16:04 DEBUG { 17:16:04 DEBUG { 17:16:04 DEBUG ----- output ----- 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG { 17:16:04 DEBUG "data": { 17:16:04 DEBUG { 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG "data": { 17:16:04 DEBUG "data": { 17:16:04 DEBUG ----- output ----- 17:16:04 DEBUG { 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG "data": { 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG "data": { 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG { 17:16:04 DEBUG "data": { 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG "data": { 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG { 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG "data": { 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "data": { 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "data": { 17:16:04 DEBUG "data": { 17:16:04 DEBUG "data": { 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "data": { 17:16:04 DEBUG "result": [ 17:16:04 DEBUG "data": { 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "result": [ 17:16:04 DEBUG "result": [ 17:16:04 DEBUG "status": "success", 17:16:04 DEBUG "data": { 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "result": [ 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "result": [ 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "data": { 17:16:04 DEBUG "result": [ 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG { 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "result": [ 17:16:04 DEBUG { 17:16:04 DEBUG { 17:16:04 DEBUG "data": { 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "result": [ 17:16:04 DEBUG { 17:16:04 DEBUG "result": [ 17:16:04 DEBUG { 17:16:04 DEBUG "result": [ 17:16:04 DEBUG "result": [ 17:16:04 DEBUG "result": [ 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG { 17:16:04 DEBUG "result": [ 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "result": [ 17:16:04 DEBUG { 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "resultType": "vector", 17:16:04 DEBUG "result": [ 17:16:04 DEBUG { 17:16:04 DEBUG "metric": { 17:16:04 DEBUG { 17:16:04 DEBUG "metric": { 17:16:04 DEBUG { 17:16:04 DEBUG { 17:16:04 DEBUG { 17:16:04 DEBUG "result": [ 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "pod": "ds-cts-0" 17:16:04 DEBUG { 17:16:04 DEBUG { 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "instance": "10.142.0.107:9100" 17:16:04 DEBUG "instance": "10.142.0.107:9100" 17:16:04 DEBUG "result": [ 17:16:04 DEBUG { 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "pod": "ds-cts-0" 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "metric": { 17:16:04 DEBUG { 17:16:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:16:04 DEBUG }, 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:16:04 DEBUG }, 17:16:04 DEBUG }, 17:16:04 DEBUG { 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:16:04 DEBUG }, 17:16:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:16:04 DEBUG }, 17:16:04 DEBUG "instance": "10.142.0.107:9100" 17:16:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:16:04 DEBUG "pod": "ds-idrepo-0" 17:16:04 DEBUG "metric": { 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:16:04 DEBUG "pod": "ds-cts-0" 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG "value": [ 17:16:04 DEBUG "metric": { 17:16:04 DEBUG "pod": "ds-idrepo-0" 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG }, 17:16:04 DEBUG }, 17:16:04 DEBUG }, 17:16:04 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 17:16:04 DEBUG }, 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG "pod": "ds-idrepo-0" 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 17:16:04 DEBUG "value": [ 17:16:04 DEBUG "value": [ 17:16:04 DEBUG "value": [ 17:16:04 DEBUG }, 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG "value": [ 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG [...] 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG "pod": "ds-cts-2" 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG 17:16:04 DEBUG [...] 17:16:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:16:04 DEBUG }, 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:16:04 DEBUG "instance": "10.142.0.43:9100" 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG "pod": "ds-cts-2" 17:16:04 DEBUG 17:16:04 DEBUG "pod": "idm-b85f9d978-txpwv" 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG "pod": "ds-idrepo-2" 17:16:04 DEBUG 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 17:16:04 DEBUG "instance": "10.142.0.43:9100" 17:16:04 DEBUG }, 17:16:04 DEBUG }, 17:16:04 DEBUG 17:16:04 DEBUG 17:16:04 DEBUG "pod": "idm-b85f9d978-txpwv" 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG }, 17:16:04 DEBUG "instance": "10.142.0.43:9100" 17:16:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:16:04 DEBUG 17:16:04 DEBUG }, 17:16:04 DEBUG 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 17:16:04 DEBUG "pod": "ds-idrepo-2" 17:16:04 DEBUG }, 17:16:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG }, 17:16:04 DEBUG }, 17:16:04 DEBUG 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG "0.23333333333333334" 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG "pod": "ds-idrepo-2" 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG }, 17:16:04 DEBUG "0.0004145466666666666" 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG "value": [ 17:16:04 DEBUG "pod": "idm-b85f9d978-txpwv" 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG "pod": "ds-cts-2" 17:16:04 DEBUG "2327.2666666666664" 17:16:04 DEBUG ] 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG "977600512" 17:16:04 DEBUG "0.061333333333377975" 17:16:04 DEBUG }, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG "value": [ 17:16:04 DEBUG ] 17:16:04 DEBUG "0" 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG }, 17:16:04 DEBUG "0" 17:16:04 DEBUG }, 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG "56577925.06666666" 17:16:04 DEBUG ] 17:16:04 DEBUG ] 17:16:04 DEBUG "value": [ 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG "0.0016962002749229886" 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG "56584260.03333333" 17:16:04 DEBUG "2327.3333333333335" 17:16:04 DEBUG "value": [ 17:16:04 DEBUG ] 17:16:04 DEBUG "value": [ 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG "7.857741972554542" 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG "0" 17:16:04 DEBUG ] 17:16:04 DEBUG "0.015" 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG ] 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG } 17:16:04 DEBUG 1713543363, 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG ] 17:16:04 DEBUG "0" 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG "409.72688316381846" 17:16:04 DEBUG ] 17:16:04 DEBUG "0" 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG ] 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG ] 17:16:04 DEBUG ] 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:16:04 DEBUG } 17:17:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543422" 17:17:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543422" 17:17:02 INFO [http_cmd]: http status code OK 17:17:02 DEBUG --- status code --- 17:17:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543422" 17:17:02 INFO [http_cmd]: http status code OK 17:17:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:02 DEBUG --- status code --- 17:17:02 DEBUG --- http response --- 17:17:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:17:02 DEBUG --- http response --- 17:17:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:17:02 ERROR [http_cmd]: ERROR 17:17:02 ERROR --- status code --- 17:17:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:17:02 ERROR --- http response --- 17:17:02 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:17:02 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:17:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543422" 17:17:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543422" 17:17:02 INFO [http_cmd]: http status code OK 17:17:02 DEBUG --- status code --- 17:17:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:02 DEBUG --- http response --- 17:17:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:17:02 DEBUG ----- output ----- 17:17:02 DEBUG { 17:17:02 DEBUG "status": "success", 17:17:02 DEBUG "data": { 17:17:02 DEBUG "resultType": "vector", 17:17:02 DEBUG "result": [ 17:17:02 DEBUG { 17:17:02 DEBUG "metric": { 17:17:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:17:02 DEBUG }, 17:17:02 DEBUG "value": [ 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG [...] 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:17:02 DEBUG }, 17:17:02 DEBUG "value": [ 17:17:02 DEBUG 1713543422, 17:17:02 DEBUG "2333.3333333333335" 17:17:02 DEBUG ] 17:17:02 DEBUG } 17:17:02 DEBUG ] 17:17:02 DEBUG } 17:17:02 DEBUG } 17:17:02 INFO [http_cmd]: http status code OK 17:17:02 DEBUG --- status code --- 17:17:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:02 DEBUG --- http response --- 17:17:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:17:02 DEBUG ----- output ----- 17:17:02 DEBUG { 17:17:02 DEBUG "status": "success", 17:17:02 DEBUG "data": { 17:17:02 DEBUG "resultType": "vector", 17:17:02 DEBUG "result": [ 17:17:02 DEBUG { 17:17:02 DEBUG "metric": { 17:17:02 DEBUG "pod": "ds-idrepo-0" 17:17:02 DEBUG }, 17:17:02 DEBUG "value": [ 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG [...] 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG "pod": "ds-cts-2" 17:17:02 DEBUG }, 17:17:02 DEBUG "value": [ 17:17:02 DEBUG 1713543422, 17:17:02 DEBUG "0" 17:17:02 DEBUG ] 17:17:02 DEBUG } 17:17:02 DEBUG ] 17:17:02 DEBUG } 17:17:02 DEBUG } 17:17:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543422" 17:17:02 INFO [http_cmd]: http status code OK 17:17:02 DEBUG --- status code --- 17:17:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:02 DEBUG --- http response --- 17:17:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:17:02 DEBUG ----- output ----- 17:17:02 DEBUG { 17:17:02 DEBUG "status": "success", 17:17:02 DEBUG "data": { 17:17:02 DEBUG "resultType": "vector", 17:17:02 DEBUG "result": [ 17:17:02 DEBUG { 17:17:02 DEBUG "metric": { 17:17:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:17:02 DEBUG }, 17:17:02 DEBUG "value": [ 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG [...] 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:17:02 DEBUG }, 17:17:02 DEBUG "value": [ 17:17:02 DEBUG 1713543422, 17:17:02 DEBUG "0" 17:17:02 DEBUG ] 17:17:02 DEBUG } 17:17:02 DEBUG ] 17:17:02 DEBUG } 17:17:02 DEBUG } 17:17:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543422" 17:17:02 INFO [http_cmd]: http status code OK 17:17:02 DEBUG --- status code --- 17:17:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:02 DEBUG --- http response --- 17:17:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:17:02 DEBUG ----- output ----- 17:17:02 DEBUG { 17:17:02 DEBUG "status": "success", 17:17:02 DEBUG "data": { 17:17:02 DEBUG "resultType": "vector", 17:17:02 DEBUG "result": [ 17:17:02 DEBUG { 17:17:02 DEBUG "metric": { 17:17:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:17:02 DEBUG }, 17:17:02 DEBUG "value": [ 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG [...] 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:17:02 DEBUG }, 17:17:02 DEBUG "value": [ 17:17:02 DEBUG 1713543422, 17:17:02 DEBUG "0" 17:17:02 DEBUG ] 17:17:02 DEBUG } 17:17:02 DEBUG ] 17:17:02 DEBUG } 17:17:02 DEBUG } 17:17:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543422" 17:17:02 INFO [http_cmd]: http status code OK 17:17:02 DEBUG --- status code --- 17:17:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:02 DEBUG --- http response --- 17:17:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:17:02 DEBUG ----- output ----- 17:17:02 DEBUG { 17:17:02 DEBUG "status": "success", 17:17:02 DEBUG "data": { 17:17:02 DEBUG "resultType": "vector", 17:17:02 DEBUG "result": [ 17:17:02 DEBUG { 17:17:02 DEBUG "metric": { 17:17:02 DEBUG "instance": "10.142.0.107:9100" 17:17:02 DEBUG }, 17:17:02 DEBUG "value": [ 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG [...] 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG 17:17:02 DEBUG "instance": "10.142.0.43:9100" 17:17:02 DEBUG }, 17:17:02 DEBUG "value": [ 17:17:02 DEBUG 1713543422, 17:17:02 DEBUG "0.04166666666666477" 17:17:02 DEBUG ] 17:17:02 DEBUG } 17:17:02 DEBUG ] 17:17:02 DEBUG } 17:17:02 DEBUG } 17:17:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543423" 17:17:03 INFO [http_cmd]: http status code OK 17:17:03 DEBUG --- status code --- 17:17:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:03 DEBUG --- http response --- 17:17:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:17:03 DEBUG ----- output ----- 17:17:03 DEBUG { 17:17:03 DEBUG "status": "success", 17:17:03 DEBUG "data": { 17:17:03 DEBUG "resultType": "vector", 17:17:03 DEBUG "result": [ 17:17:03 DEBUG { 17:17:03 DEBUG "metric": { 17:17:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:17:03 DEBUG }, 17:17:03 DEBUG "value": [ 17:17:03 DEBUG 17:17:03 DEBUG 17:17:03 DEBUG 17:17:03 DEBUG [...] 17:17:03 DEBUG 17:17:03 DEBUG 17:17:03 DEBUG 17:17:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:17:03 DEBUG }, 17:17:03 DEBUG "value": [ 17:17:03 DEBUG 1713543423, 17:17:03 DEBUG "2347147" 17:17:03 DEBUG ] 17:17:03 DEBUG } 17:17:03 DEBUG ] 17:17:03 DEBUG } 17:17:03 DEBUG } 17:17:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543423" 17:17:03 INFO [http_cmd]: http status code OK 17:17:03 DEBUG --- status code --- 17:17:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:03 DEBUG --- http response --- 17:17:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:17:03 DEBUG ----- output ----- 17:17:03 DEBUG { 17:17:03 DEBUG "status": "success", 17:17:03 DEBUG "data": { 17:17:03 DEBUG "resultType": "vector", 17:17:03 DEBUG "result": [ 17:17:03 DEBUG { 17:17:03 DEBUG "metric": { 17:17:03 DEBUG "instance": "10.142.0.107:9100" 17:17:03 DEBUG }, 17:17:03 DEBUG "value": [ 17:17:03 DEBUG 17:17:03 DEBUG 17:17:03 DEBUG 17:17:03 DEBUG [...] 17:17:03 DEBUG 17:17:03 DEBUG 17:17:03 DEBUG 17:17:03 DEBUG "instance": "10.142.0.43:9100" 17:17:03 DEBUG }, 17:17:03 DEBUG "value": [ 17:17:03 DEBUG 1713543423, 17:17:03 DEBUG "1671949587456" 17:17:03 DEBUG ] 17:17:03 DEBUG } 17:17:03 DEBUG ] 17:17:03 DEBUG } 17:17:03 DEBUG } 17:17:03 INFO 17:17:03 INFO [loop_until]: kubectl --namespace=xlou top node 17:17:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:17:03 INFO [loop_until]: OK (rc = 0) 17:17:03 DEBUG --- stdout --- 17:17:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6174Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5389m 18% 6968Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 47m 0% 2674Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11168m 37% 11995Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9162m 30% 15449Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 133m 0% 5230Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11393m 38% 11638Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12248m 41% 2953Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12689m 42% 10981Mi 9% gke-xlou-cdm-ds-0f601751-j488 5978m 20% 6637Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9353m 31% 17169Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6258m 20% 7184Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9290m 31% 14886Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8016m 50% 4629Mi 7% 17:17:03 DEBUG --- stderr --- 17:17:03 DEBUG 17:17:04 INFO 17:17:04 INFO [loop_until]: kubectl --namespace=xlou top pods 17:17:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543424" 17:17:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543424" 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 INFO [loop_until]: OK (rc = 0) 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 DEBUG --- status code --- 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG --- status code --- 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 DEBUG --- stdout --- 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG --- status code --- 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG --- status code --- 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 INFO [http_cmd]: http status code OK 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11038m 8716Mi am-6fd554f8f7-btd4s 11247m 8764Mi am-6fd554f8f7-wxcsj 11798m 8772Mi ds-cts-0 8990m 12432Mi ds-cts-1 9172m 12367Mi ds-cts-2 9216m 12418Mi ds-idrepo-0 5656m 3414Mi ds-idrepo-1 5572m 3365Mi ds-idrepo-2 5179m 3423Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7834m 843Mi 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG --- status code --- 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG --- stderr --- 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG --- http response --- 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG { 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:17:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG { 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG { 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG ----- output ----- 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "data": { 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG { 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG "data": { 17:17:04 DEBUG "data": { 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "data": { 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "data": { 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "data": { 17:17:04 DEBUG "data": { 17:17:04 DEBUG "data": { 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "data": { 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "status": "success", 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "data": { 17:17:04 DEBUG "data": { 17:17:04 DEBUG "data": { 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "data": { 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "data": { 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "data": { 17:17:04 DEBUG "data": { 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "data": { 17:17:04 DEBUG { 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "result": [ 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG { 17:17:04 DEBUG "resultType": "vector", 17:17:04 DEBUG "metric": { 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "result": [ 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG "metric": { 17:17:04 DEBUG "metric": { 17:17:04 DEBUG "result": [ 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG "result": [ 17:17:04 DEBUG { 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "metric": { 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "instance": "10.142.0.107:9100" 17:17:04 DEBUG { 17:17:04 DEBUG "result": [ 17:17:04 DEBUG "metric": { 17:17:04 DEBUG "metric": { 17:17:04 DEBUG "pod": "ds-cts-0" 17:17:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:17:04 DEBUG { 17:17:04 DEBUG "metric": { 17:17:04 DEBUG "metric": { 17:17:04 DEBUG "metric": { 17:17:04 DEBUG { 17:17:04 DEBUG "metric": { 17:17:04 DEBUG { 17:17:04 DEBUG { 17:17:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:17:04 DEBUG { 17:17:04 DEBUG }, 17:17:04 DEBUG "metric": { 17:17:04 DEBUG { 17:17:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:17:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:17:04 DEBUG }, 17:17:04 DEBUG }, 17:17:04 DEBUG "metric": { 17:17:04 DEBUG "pod": "ds-idrepo-0" 17:17:04 DEBUG "pod": "ds-idrepo-0" 17:17:04 DEBUG "instance": "10.142.0.107:9100" 17:17:04 DEBUG }, 17:17:04 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:17:04 DEBUG "metric": { 17:17:04 DEBUG "metric": { 17:17:04 DEBUG }, 17:17:04 DEBUG "metric": { 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "pod": "ds-cts-0" 17:17:04 DEBUG "metric": { 17:17:04 DEBUG }, 17:17:04 DEBUG }, 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:17:04 DEBUG }, 17:17:04 DEBUG }, 17:17:04 DEBUG "metric": { 17:17:04 DEBUG "value": [ 17:17:04 DEBUG }, 17:17:04 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:17:04 DEBUG "instance": "10.142.0.107:9100" 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 17:17:04 DEBUG "pod": "ds-cts-0" 17:17:04 DEBUG }, 17:17:04 DEBUG "pod": "ds-idrepo-0" 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG }, 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 17:17:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:17:04 DEBUG 17:17:04 DEBUG "value": [ 17:17:04 DEBUG }, 17:17:04 DEBUG }, 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG }, 17:17:04 DEBUG "value": [ 17:17:04 DEBUG }, 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 17:17:04 DEBUG }, 17:17:04 DEBUG 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG [...] 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG "pod": "ds-cts-2" 17:17:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:17:04 DEBUG 17:17:04 DEBUG "pod": "ds-idrepo-2" 17:17:04 DEBUG 17:17:04 DEBUG "instance": "10.142.0.43:9100" 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG "instance": "10.142.0.43:9100" 17:17:04 DEBUG }, 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG 17:17:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:17:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:17:04 DEBUG }, 17:17:04 DEBUG }, 17:17:04 DEBUG 17:17:04 DEBUG }, 17:17:04 DEBUG 17:17:04 DEBUG }, 17:17:04 DEBUG 17:17:04 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG }, 17:17:04 DEBUG 17:17:04 DEBUG 17:17:04 DEBUG "2333.233333333333" 17:17:04 DEBUG 17:17:04 DEBUG }, 17:17:04 DEBUG }, 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "pod": "ds-idrepo-2" 17:17:04 DEBUG }, 17:17:04 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:17:04 DEBUG "instance": "10.142.0.43:9100" 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "pod": "ds-cts-2" 17:17:04 DEBUG 17:17:04 DEBUG ] 17:17:04 DEBUG "pod": "ds-cts-2" 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG }, 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG }, 17:17:04 DEBUG "value": [ 17:17:04 DEBUG }, 17:17:04 DEBUG }, 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG }, 17:17:04 DEBUG "pod": "ds-idrepo-2" 17:17:04 DEBUG } 17:17:04 DEBUG }, 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG "0" 17:17:04 DEBUG "7.882014692259078" 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "0" 17:17:04 DEBUG }, 17:17:04 DEBUG "56655686.96666667" 17:17:04 DEBUG "value": [ 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "0.06313333333336003" 17:17:04 DEBUG "value": [ 17:17:04 DEBUG }, 17:17:04 DEBUG ] 17:17:04 DEBUG "value": [ 17:17:04 DEBUG "0.0165" 17:17:04 DEBUG "978505728" 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG ] 17:17:04 DEBUG "value": [ 17:17:04 DEBUG ] 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG "0" 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG ] 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG "value": [ 17:17:04 DEBUG } 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG "0.000018393209852981785" 17:17:04 DEBUG } 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG } 17:17:04 DEBUG "0" 17:17:04 DEBUG ] 17:17:04 DEBUG "0" 17:17:04 DEBUG "56649240.4" 17:17:04 DEBUG } 17:17:04 DEBUG "0" 17:17:04 DEBUG 1713543424, 17:17:04 DEBUG } 17:17:04 DEBUG "0.8" 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG "2333.233333333333" 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG } 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG "0" 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG ] 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG ] 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG } 17:17:04 DEBUG ] 17:17:04 DEBUG } 17:17:04 DEBUG ] 17:17:04 DEBUG ] 17:17:04 DEBUG } 17:17:04 DEBUG ] 17:17:04 DEBUG } 17:17:04 DEBUG ] 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG ] 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG ] 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:17:04 DEBUG } 17:18:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543482" 17:18:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543482" 17:18:02 INFO [http_cmd]: http status code OK 17:18:02 DEBUG --- status code --- 17:18:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:02 DEBUG --- http response --- 17:18:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:18:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543482" 17:18:02 INFO [http_cmd]: http status code OK 17:18:02 DEBUG --- status code --- 17:18:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:02 DEBUG --- http response --- 17:18:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:18:02 ERROR [http_cmd]: ERROR 17:18:02 ERROR --- status code --- 17:18:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:18:02 ERROR --- http response --- 17:18:02 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:18:02 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:18:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543482" 17:18:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543482" 17:18:02 INFO [http_cmd]: http status code OK 17:18:02 DEBUG --- status code --- 17:18:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:02 DEBUG --- http response --- 17:18:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:18:02 DEBUG ----- output ----- 17:18:02 DEBUG { 17:18:02 DEBUG "status": "success", 17:18:02 DEBUG "data": { 17:18:02 DEBUG "resultType": "vector", 17:18:02 DEBUG "result": [ 17:18:02 DEBUG { 17:18:02 DEBUG "metric": { 17:18:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:18:02 DEBUG }, 17:18:02 DEBUG "value": [ 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG [...] 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:18:02 DEBUG }, 17:18:02 DEBUG "value": [ 17:18:02 DEBUG 1713543482, 17:18:02 DEBUG "2324.3333333333335" 17:18:02 DEBUG ] 17:18:02 DEBUG } 17:18:02 DEBUG ] 17:18:02 DEBUG } 17:18:02 DEBUG } 17:18:02 INFO [http_cmd]: http status code OK 17:18:02 DEBUG --- status code --- 17:18:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:02 DEBUG --- http response --- 17:18:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:18:02 DEBUG ----- output ----- 17:18:02 DEBUG { 17:18:02 DEBUG "status": "success", 17:18:02 DEBUG "data": { 17:18:02 DEBUG "resultType": "vector", 17:18:02 DEBUG "result": [ 17:18:02 DEBUG { 17:18:02 DEBUG "metric": { 17:18:02 DEBUG "pod": "ds-idrepo-0" 17:18:02 DEBUG }, 17:18:02 DEBUG "value": [ 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG [...] 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG "pod": "ds-cts-2" 17:18:02 DEBUG }, 17:18:02 DEBUG "value": [ 17:18:02 DEBUG 1713543482, 17:18:02 DEBUG "0" 17:18:02 DEBUG ] 17:18:02 DEBUG } 17:18:02 DEBUG ] 17:18:02 DEBUG } 17:18:02 DEBUG } 17:18:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543482" 17:18:02 INFO [http_cmd]: http status code OK 17:18:02 DEBUG --- status code --- 17:18:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:02 DEBUG --- http response --- 17:18:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:18:02 DEBUG ----- output ----- 17:18:02 DEBUG { 17:18:02 DEBUG "status": "success", 17:18:02 DEBUG "data": { 17:18:02 DEBUG "resultType": "vector", 17:18:02 DEBUG "result": [ 17:18:02 DEBUG { 17:18:02 DEBUG "metric": { 17:18:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:18:02 DEBUG }, 17:18:02 DEBUG "value": [ 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG [...] 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:18:02 DEBUG }, 17:18:02 DEBUG "value": [ 17:18:02 DEBUG 1713543482, 17:18:02 DEBUG "0" 17:18:02 DEBUG ] 17:18:02 DEBUG } 17:18:02 DEBUG ] 17:18:02 DEBUG } 17:18:02 DEBUG } 17:18:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543482" 17:18:02 INFO [http_cmd]: http status code OK 17:18:02 DEBUG --- status code --- 17:18:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:02 DEBUG --- http response --- 17:18:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:18:02 DEBUG ----- output ----- 17:18:02 DEBUG { 17:18:02 DEBUG "status": "success", 17:18:02 DEBUG "data": { 17:18:02 DEBUG "resultType": "vector", 17:18:02 DEBUG "result": [ 17:18:02 DEBUG { 17:18:02 DEBUG "metric": { 17:18:02 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:18:02 DEBUG }, 17:18:02 DEBUG "value": [ 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG [...] 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG 17:18:02 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:18:02 DEBUG }, 17:18:02 DEBUG "value": [ 17:18:02 DEBUG 1713543482, 17:18:02 DEBUG "0" 17:18:02 DEBUG ] 17:18:02 DEBUG } 17:18:02 DEBUG ] 17:18:02 DEBUG } 17:18:02 DEBUG } 17:18:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543482" 17:18:03 INFO [http_cmd]: http status code OK 17:18:03 DEBUG --- status code --- 17:18:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:03 DEBUG --- http response --- 17:18:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:18:03 DEBUG ----- output ----- 17:18:03 DEBUG { 17:18:03 DEBUG "status": "success", 17:18:03 DEBUG "data": { 17:18:03 DEBUG "resultType": "vector", 17:18:03 DEBUG "result": [ 17:18:03 DEBUG { 17:18:03 DEBUG "metric": { 17:18:03 DEBUG "instance": "10.142.0.107:9100" 17:18:03 DEBUG }, 17:18:03 DEBUG "value": [ 17:18:03 DEBUG 17:18:03 DEBUG 17:18:03 DEBUG 17:18:03 DEBUG [...] 17:18:03 DEBUG 17:18:03 DEBUG 17:18:03 DEBUG 17:18:03 DEBUG "instance": "10.142.0.43:9100" 17:18:03 DEBUG }, 17:18:03 DEBUG "value": [ 17:18:03 DEBUG 1713543482, 17:18:03 DEBUG "0.037999999999993865" 17:18:03 DEBUG ] 17:18:03 DEBUG } 17:18:03 DEBUG ] 17:18:03 DEBUG } 17:18:03 DEBUG } 17:18:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543483" 17:18:03 INFO [http_cmd]: http status code OK 17:18:03 DEBUG --- status code --- 17:18:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:03 DEBUG --- http response --- 17:18:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:18:03 DEBUG ----- output ----- 17:18:03 DEBUG { 17:18:03 DEBUG "status": "success", 17:18:03 DEBUG "data": { 17:18:03 DEBUG "resultType": "vector", 17:18:03 DEBUG "result": [ 17:18:03 DEBUG { 17:18:03 DEBUG "metric": { 17:18:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:18:03 DEBUG }, 17:18:03 DEBUG "value": [ 17:18:03 DEBUG 17:18:03 DEBUG 17:18:03 DEBUG 17:18:03 DEBUG [...] 17:18:03 DEBUG 17:18:03 DEBUG 17:18:03 DEBUG 17:18:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:18:03 DEBUG }, 17:18:03 DEBUG "value": [ 17:18:03 DEBUG 1713543483, 17:18:03 DEBUG "2486636" 17:18:03 DEBUG ] 17:18:03 DEBUG } 17:18:03 DEBUG ] 17:18:03 DEBUG } 17:18:03 DEBUG } 17:18:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543483" 17:18:03 INFO [http_cmd]: http status code OK 17:18:03 DEBUG --- status code --- 17:18:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:03 DEBUG --- http response --- 17:18:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:18:03 DEBUG ----- output ----- 17:18:03 DEBUG { 17:18:03 DEBUG "status": "success", 17:18:03 DEBUG "data": { 17:18:03 DEBUG "resultType": "vector", 17:18:03 DEBUG "result": [ 17:18:03 DEBUG { 17:18:03 DEBUG "metric": { 17:18:03 DEBUG "instance": "10.142.0.107:9100" 17:18:03 DEBUG }, 17:18:03 DEBUG "value": [ 17:18:03 DEBUG 17:18:03 DEBUG 17:18:03 DEBUG 17:18:03 DEBUG [...] 17:18:03 DEBUG 17:18:03 DEBUG 17:18:03 DEBUG 17:18:03 DEBUG "instance": "10.142.0.43:9100" 17:18:03 DEBUG }, 17:18:03 DEBUG "value": [ 17:18:03 DEBUG 1713543483, 17:18:03 DEBUG "1674751400960" 17:18:03 DEBUG ] 17:18:03 DEBUG } 17:18:03 DEBUG ] 17:18:03 DEBUG } 17:18:03 DEBUG } 17:18:03 INFO 17:18:03 INFO [loop_until]: kubectl --namespace=xlou top node 17:18:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:18:04 INFO [loop_until]: OK (rc = 0) 17:18:04 DEBUG --- stdout --- 17:18:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6183Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5362m 17% 6994Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 47m 0% 2689Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11152m 37% 12002Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9023m 30% 15452Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5222Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11205m 37% 11632Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12235m 40% 2960Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12863m 43% 10975Mi 9% gke-xlou-cdm-ds-0f601751-j488 6221m 20% 6649Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9226m 30% 17171Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6031m 20% 7182Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9304m 31% 14896Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8003m 50% 4633Mi 7% 17:18:04 DEBUG --- stderr --- 17:18:04 DEBUG 17:18:04 INFO 17:18:04 INFO [loop_until]: kubectl --namespace=xlou top pods 17:18:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543484" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543484" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543484" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543484" 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG { 17:18:04 DEBUG "status": "success", 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 DEBUG "data": { 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG "result": [ 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG { 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:18:04 DEBUG "metric": { 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG "pod": "ds-idrepo-0" 17:18:04 DEBUG { 17:18:04 DEBUG }, 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "data": { 17:18:04 DEBUG 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG 17:18:04 DEBUG "result": [ 17:18:04 DEBUG 17:18:04 DEBUG { 17:18:04 DEBUG [...] 17:18:04 DEBUG "metric": { 17:18:04 DEBUG 17:18:04 DEBUG "pod": "ds-idrepo-0" 17:18:04 DEBUG 17:18:04 DEBUG }, 17:18:04 DEBUG 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "pod": "ds-idrepo-2" 17:18:04 DEBUG }, 17:18:04 DEBUG 17:18:04 DEBUG "value": [ 17:18:04 DEBUG 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG "0" 17:18:04 DEBUG [...] 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG ] 17:18:04 DEBUG 17:18:04 DEBUG } 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 DEBUG 17:18:04 DEBUG ] 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG 17:18:04 DEBUG } 17:18:04 DEBUG { 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG "pod": "ds-idrepo-2" 17:18:04 DEBUG }, 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "data": { 17:18:04 DEBUG } 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG "0" 17:18:04 DEBUG "result": [ 17:18:04 DEBUG { 17:18:04 DEBUG ] 17:18:04 DEBUG { 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543484" 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG } 17:18:04 DEBUG "metric": { 17:18:04 DEBUG "data": { 17:18:04 DEBUG ] 17:18:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG "result": [ 17:18:04 DEBUG }, 17:18:04 DEBUG { 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "metric": { 17:18:04 DEBUG 17:18:04 DEBUG "pod": "ds-cts-0" 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG }, 17:18:04 DEBUG [...] 17:18:04 DEBUG "value": [ 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:18:04 DEBUG [...] 17:18:04 DEBUG }, 17:18:04 DEBUG 17:18:04 DEBUG "value": [ 17:18:04 DEBUG 17:18:04 DEBUG 1713543484, 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543484" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543484" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543484" 17:18:04 DEBUG 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543484" 17:18:04 DEBUG "981590016" 17:18:04 DEBUG "pod": "ds-cts-2" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543484" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543484" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543484" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543484" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543484" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543484" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543484" 17:18:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543484" 17:18:04 DEBUG ] 17:18:04 DEBUG }, 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 DEBUG } 17:18:04 DEBUG "value": [ 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 INFO [loop_until]: OK (rc = 0) 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG ] 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG --- stdout --- 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG --- status code --- 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG } 17:18:04 DEBUG "0" 17:18:04 INFO [http_cmd]: http status code OK 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11107m 8716Mi am-6fd554f8f7-btd4s 11456m 8766Mi am-6fd554f8f7-wxcsj 11875m 8774Mi ds-cts-0 9182m 12435Mi ds-cts-1 9220m 12374Mi ds-cts-2 9490m 12418Mi ds-idrepo-0 5616m 3413Mi ds-idrepo-1 5554m 3366Mi ds-idrepo-2 5429m 3424Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7930m 845Mi 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG } 17:18:04 DEBUG ] 17:18:04 DEBUG } 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG --- stderr --- 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:18:04 DEBUG --- status code --- 17:18:04 DEBUG ] 17:18:04 DEBUG 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:18:04 DEBUG } 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:18:04 DEBUG { 17:18:04 DEBUG { 17:18:04 DEBUG { 17:18:04 DEBUG { 17:18:04 DEBUG { 17:18:04 DEBUG --- http response --- 17:18:04 DEBUG } 17:18:04 DEBUG { 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG { 17:18:04 DEBUG { 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG "data": { 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG { 17:18:04 DEBUG { 17:18:04 DEBUG { 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG { 17:18:04 DEBUG "data": { 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG "data": { 17:18:04 DEBUG "data": { 17:18:04 DEBUG ----- output ----- 17:18:04 DEBUG "data": { 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG "data": { 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG "data": { 17:18:04 DEBUG "data": { 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG "result": [ 17:18:04 DEBUG "data": { 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG { 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG "data": { 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG "data": { 17:18:04 DEBUG "result": [ 17:18:04 DEBUG { 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG "result": [ 17:18:04 DEBUG "result": [ 17:18:04 DEBUG "status": "success", 17:18:04 DEBUG "result": [ 17:18:04 DEBUG "data": { 17:18:04 DEBUG "result": [ 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG "result": [ 17:18:04 DEBUG "result": [ 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG { 17:18:04 DEBUG "metric": { 17:18:04 DEBUG "result": [ 17:18:04 DEBUG { 17:18:04 DEBUG { 17:18:04 DEBUG "data": { 17:18:04 DEBUG { 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG { 17:18:04 DEBUG "result": [ 17:18:04 DEBUG { 17:18:04 DEBUG { 17:18:04 DEBUG "result": [ 17:18:04 DEBUG "metric": { 17:18:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:18:04 DEBUG { 17:18:04 DEBUG "metric": { 17:18:04 DEBUG "metric": { 17:18:04 DEBUG "resultType": "vector", 17:18:04 DEBUG "metric": { 17:18:04 DEBUG "result": [ 17:18:04 DEBUG "metric": { 17:18:04 DEBUG { 17:18:04 DEBUG "metric": { 17:18:04 DEBUG "metric": { 17:18:04 DEBUG { 17:18:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:18:04 DEBUG }, 17:18:04 DEBUG "metric": { 17:18:04 DEBUG "pod": "ds-cts-0" 17:18:04 DEBUG "instance": "10.142.0.107:9100" 17:18:04 DEBUG "result": [ 17:18:04 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:18:04 DEBUG { 17:18:04 DEBUG "pod": "ds-cts-0" 17:18:04 DEBUG "metric": { 17:18:04 DEBUG "instance": "10.142.0.107:9100" 17:18:04 DEBUG "instance": "10.142.0.107:9100" 17:18:04 DEBUG "metric": { 17:18:04 DEBUG }, 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:18:04 DEBUG }, 17:18:04 DEBUG }, 17:18:04 DEBUG { 17:18:04 DEBUG }, 17:18:04 DEBUG "metric": { 17:18:04 DEBUG }, 17:18:04 DEBUG }, 17:18:04 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:18:04 DEBUG }, 17:18:04 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:18:04 DEBUG "value": [ 17:18:04 DEBUG 17:18:04 DEBUG }, 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "metric": { 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "value": [ 17:18:04 DEBUG }, 17:18:04 DEBUG "value": [ 17:18:04 DEBUG }, 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG "value": [ 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG "pod": "ds-idrepo-0" 17:18:04 DEBUG 17:18:04 DEBUG }, 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG "value": [ 17:18:04 DEBUG 17:18:04 DEBUG "value": [ 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG }, 17:18:04 DEBUG 17:18:04 DEBUG "value": [ 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG [...] 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG "value": [ 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG [...] 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG [...] 17:18:04 DEBUG [...] 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG [...] 17:18:04 DEBUG [...] 17:18:04 DEBUG 17:18:04 DEBUG [...] 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG [...] 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG [...] 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG [...] 17:18:04 DEBUG 17:18:04 DEBUG [...] 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG [...] 17:18:04 DEBUG [...] 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:18:04 DEBUG }, 17:18:04 DEBUG 17:18:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:18:04 DEBUG "instance": "10.142.0.43:9100" 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG "pod": "ds-cts-2" 17:18:04 DEBUG "instance": "10.142.0.43:9100" 17:18:04 DEBUG 17:18:04 DEBUG "instance": "10.142.0.43:9100" 17:18:04 DEBUG 17:18:04 DEBUG }, 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "pod": "ds-cts-2" 17:18:04 DEBUG }, 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "pod": "ds-idrepo-2" 17:18:04 DEBUG 17:18:04 DEBUG 17:18:04 DEBUG }, 17:18:04 DEBUG }, 17:18:04 DEBUG "pod": "ds-idrepo-2" 17:18:04 DEBUG }, 17:18:04 DEBUG "pod": "ds-idrepo-2" 17:18:04 DEBUG "value": [ 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG }, 17:18:04 DEBUG }, 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG }, 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "value": [ 17:18:04 DEBUG }, 17:18:04 DEBUG "value": [ 17:18:04 DEBUG }, 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG "2324.1666666666665" 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "7.826690202977022" 17:18:04 DEBUG "pod": "ds-idrepo-2" 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG }, 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG "value": [ 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "0.018" 17:18:04 DEBUG ] 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG ] 17:18:04 DEBUG }, 17:18:04 DEBUG "0" 17:18:04 DEBUG "value": [ 17:18:04 DEBUG "0.8" 17:18:04 DEBUG "56324953.266666666" 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG "0.06380000000007537" 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG ] 17:18:04 DEBUG } 17:18:04 DEBUG "0.00005599333333333334" 17:18:04 DEBUG "56330932.86666667" 17:18:04 DEBUG } 17:18:04 DEBUG "value": [ 17:18:04 DEBUG ] 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG ] 17:18:04 DEBUG ] 17:18:04 DEBUG "4717.806956922368" 17:18:04 DEBUG ] 17:18:04 DEBUG "5.412246092078444" 17:18:04 DEBUG } 17:18:04 DEBUG ] 17:18:04 DEBUG ] 17:18:04 DEBUG ] 17:18:04 DEBUG ] 17:18:04 DEBUG 1713543484, 17:18:04 DEBUG } 17:18:04 DEBUG "2324.233333333333" 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG ] 17:18:04 DEBUG } 17:18:04 DEBUG ] 17:18:04 DEBUG ] 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG "0" 17:18:04 DEBUG ] 17:18:04 DEBUG ] 17:18:04 DEBUG ] 17:18:04 DEBUG ] 17:18:04 DEBUG } 17:18:04 DEBUG ] 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG ] 17:18:04 DEBUG ] 17:18:04 DEBUG } 17:18:04 DEBUG ] 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG ] 17:18:04 DEBUG } 17:18:04 DEBUG ] 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG ] 17:18:04 DEBUG ] 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:18:04 DEBUG } 17:19:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543542" 17:19:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543542" 17:19:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543542" 17:19:02 INFO [http_cmd]: http status code OK 17:19:02 INFO [http_cmd]: http status code OK 17:19:02 DEBUG --- status code --- 17:19:02 DEBUG --- status code --- 17:19:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:02 DEBUG --- http response --- 17:19:02 DEBUG --- http response --- 17:19:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:19:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:19:02 ERROR [http_cmd]: ERROR 17:19:02 ERROR --- status code --- 17:19:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:19:02 ERROR --- http response --- 17:19:02 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:19:02 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:19:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543542" 17:19:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543542" 17:19:02 INFO [http_cmd]: http status code OK 17:19:02 INFO [http_cmd]: http status code OK 17:19:02 DEBUG --- status code --- 17:19:02 DEBUG --- status code --- 17:19:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:02 DEBUG --- http response --- 17:19:02 DEBUG --- http response --- 17:19:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:19:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:19:02 DEBUG ----- output ----- 17:19:02 DEBUG ----- output ----- 17:19:02 DEBUG { 17:19:02 DEBUG { 17:19:02 DEBUG "status": "success", 17:19:02 DEBUG "status": "success", 17:19:02 DEBUG "data": { 17:19:02 DEBUG "data": { 17:19:02 DEBUG "resultType": "vector", 17:19:02 DEBUG "resultType": "vector", 17:19:02 DEBUG "result": [ 17:19:02 DEBUG "result": [ 17:19:02 DEBUG { 17:19:02 DEBUG { 17:19:02 DEBUG "metric": { 17:19:02 DEBUG "metric": { 17:19:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:19:02 DEBUG "pod": "ds-idrepo-0" 17:19:02 DEBUG }, 17:19:02 DEBUG }, 17:19:02 DEBUG "value": [ 17:19:02 DEBUG "value": [ 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG [...] 17:19:02 DEBUG [...] 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:19:02 DEBUG "pod": "ds-cts-2" 17:19:02 DEBUG }, 17:19:02 DEBUG }, 17:19:02 DEBUG "value": [ 17:19:02 DEBUG "value": [ 17:19:02 DEBUG 1713543542, 17:19:02 DEBUG 1713543542, 17:19:02 DEBUG "2333.1" 17:19:02 DEBUG "0" 17:19:02 DEBUG ] 17:19:02 DEBUG ] 17:19:02 DEBUG } 17:19:02 DEBUG } 17:19:02 DEBUG ] 17:19:02 DEBUG ] 17:19:02 DEBUG } 17:19:02 DEBUG } 17:19:02 DEBUG } 17:19:02 DEBUG } 17:19:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543542" 17:19:02 INFO [http_cmd]: http status code OK 17:19:02 DEBUG --- status code --- 17:19:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:02 DEBUG --- http response --- 17:19:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:19:02 DEBUG ----- output ----- 17:19:02 DEBUG { 17:19:02 DEBUG "status": "success", 17:19:02 DEBUG "data": { 17:19:02 DEBUG "resultType": "vector", 17:19:02 DEBUG "result": [ 17:19:02 DEBUG { 17:19:02 DEBUG "metric": { 17:19:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:19:02 DEBUG }, 17:19:02 DEBUG "value": [ 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG [...] 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG 17:19:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:19:02 DEBUG }, 17:19:02 DEBUG "value": [ 17:19:02 DEBUG 1713543542, 17:19:02 DEBUG "0" 17:19:02 DEBUG ] 17:19:02 DEBUG } 17:19:02 DEBUG ] 17:19:02 DEBUG } 17:19:02 DEBUG } 17:19:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543543" 17:19:03 INFO [http_cmd]: http status code OK 17:19:03 DEBUG --- status code --- 17:19:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:03 DEBUG --- http response --- 17:19:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:19:03 DEBUG ----- output ----- 17:19:03 DEBUG { 17:19:03 DEBUG "status": "success", 17:19:03 DEBUG "data": { 17:19:03 DEBUG "resultType": "vector", 17:19:03 DEBUG "result": [ 17:19:03 DEBUG { 17:19:03 DEBUG "metric": { 17:19:03 DEBUG "pod": "ds-idrepo-1" 17:19:03 DEBUG }, 17:19:03 DEBUG "value": [ 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG [...] 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:19:03 DEBUG }, 17:19:03 DEBUG "value": [ 17:19:03 DEBUG 1713543543, 17:19:03 DEBUG "0" 17:19:03 DEBUG ] 17:19:03 DEBUG } 17:19:03 DEBUG ] 17:19:03 DEBUG } 17:19:03 DEBUG } 17:19:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543543" 17:19:03 INFO [http_cmd]: http status code OK 17:19:03 DEBUG --- status code --- 17:19:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:03 DEBUG --- http response --- 17:19:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:19:03 DEBUG ----- output ----- 17:19:03 DEBUG { 17:19:03 DEBUG "status": "success", 17:19:03 DEBUG "data": { 17:19:03 DEBUG "resultType": "vector", 17:19:03 DEBUG "result": [ 17:19:03 DEBUG { 17:19:03 DEBUG "metric": { 17:19:03 DEBUG "instance": "10.142.0.107:9100" 17:19:03 DEBUG }, 17:19:03 DEBUG "value": [ 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG [...] 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG "instance": "10.142.0.43:9100" 17:19:03 DEBUG }, 17:19:03 DEBUG "value": [ 17:19:03 DEBUG 1713543543, 17:19:03 DEBUG "0.04266666666666765" 17:19:03 DEBUG ] 17:19:03 DEBUG } 17:19:03 DEBUG ] 17:19:03 DEBUG } 17:19:03 DEBUG } 17:19:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543543" 17:19:03 INFO [http_cmd]: http status code OK 17:19:03 DEBUG --- status code --- 17:19:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:03 DEBUG --- http response --- 17:19:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:19:03 DEBUG ----- output ----- 17:19:03 DEBUG { 17:19:03 DEBUG "status": "success", 17:19:03 DEBUG "data": { 17:19:03 DEBUG "resultType": "vector", 17:19:03 DEBUG "result": [ 17:19:03 DEBUG { 17:19:03 DEBUG "metric": { 17:19:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:19:03 DEBUG }, 17:19:03 DEBUG "value": [ 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG [...] 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:19:03 DEBUG }, 17:19:03 DEBUG "value": [ 17:19:03 DEBUG 1713543543, 17:19:03 DEBUG "2626302" 17:19:03 DEBUG ] 17:19:03 DEBUG } 17:19:03 DEBUG ] 17:19:03 DEBUG } 17:19:03 DEBUG } 17:19:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543543" 17:19:03 INFO [http_cmd]: http status code OK 17:19:03 DEBUG --- status code --- 17:19:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:03 DEBUG --- http response --- 17:19:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:19:03 DEBUG ----- output ----- 17:19:03 DEBUG { 17:19:03 DEBUG "status": "success", 17:19:03 DEBUG "data": { 17:19:03 DEBUG "resultType": "vector", 17:19:03 DEBUG "result": [ 17:19:03 DEBUG { 17:19:03 DEBUG "metric": { 17:19:03 DEBUG "instance": "10.142.0.107:9100" 17:19:03 DEBUG }, 17:19:03 DEBUG "value": [ 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG [...] 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG 17:19:03 DEBUG "instance": "10.142.0.43:9100" 17:19:03 DEBUG }, 17:19:03 DEBUG "value": [ 17:19:03 DEBUG 1713543543, 17:19:03 DEBUG "1677535290368" 17:19:03 DEBUG ] 17:19:03 DEBUG } 17:19:03 DEBUG ] 17:19:03 DEBUG } 17:19:03 DEBUG } 17:19:04 INFO 17:19:04 INFO [loop_until]: kubectl --namespace=xlou top node 17:19:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:19:04 INFO [loop_until]: OK (rc = 0) 17:19:04 DEBUG --- stdout --- 17:19:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6176Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5442m 18% 6986Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 45m 0% 2690Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11295m 37% 12000Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9243m 30% 15449Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 5231Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11523m 38% 11641Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12376m 41% 2966Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13040m 43% 10991Mi 9% gke-xlou-cdm-ds-0f601751-j488 6222m 20% 6650Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9538m 31% 17176Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6289m 21% 7178Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9556m 32% 14891Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8044m 50% 4633Mi 7% 17:19:04 DEBUG --- stderr --- 17:19:04 DEBUG 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543544" 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543544" 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG { 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "data": { 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:19:04 DEBUG "result": [ 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG { 17:19:04 DEBUG { 17:19:04 DEBUG "metric": { 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "pod": "ds-idrepo-0" 17:19:04 DEBUG "data": { 17:19:04 DEBUG }, 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG "value": [ 17:19:04 DEBUG "result": [ 17:19:04 DEBUG 17:19:04 DEBUG { 17:19:04 DEBUG 17:19:04 DEBUG "metric": { 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG "pod": "ds-idrepo-0" 17:19:04 DEBUG 17:19:04 DEBUG }, 17:19:04 DEBUG 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG "pod": "ds-idrepo-2" 17:19:04 DEBUG 17:19:04 DEBUG }, 17:19:04 DEBUG 17:19:04 DEBUG "value": [ 17:19:04 DEBUG [...] 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG 17:19:04 DEBUG "0" 17:19:04 DEBUG 17:19:04 DEBUG ] 17:19:04 DEBUG 17:19:04 DEBUG } 17:19:04 DEBUG "pod": "ds-idrepo-2" 17:19:04 DEBUG ] 17:19:04 DEBUG }, 17:19:04 DEBUG } 17:19:04 DEBUG "value": [ 17:19:04 DEBUG } 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG "0" 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543544" 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG { 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "data": { 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG "result": [ 17:19:04 DEBUG { 17:19:04 DEBUG "metric": { 17:19:04 DEBUG "pod": "ds-cts-0" 17:19:04 DEBUG }, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG "pod": "ds-cts-2" 17:19:04 DEBUG }, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG "0.0007811133333333333" 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543544" 17:19:04 INFO 17:19:04 INFO [loop_until]: kubectl --namespace=xlou top pods 17:19:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543544" 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG { 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "data": { 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG "result": [ 17:19:04 DEBUG { 17:19:04 DEBUG "metric": { 17:19:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:19:04 DEBUG }, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:19:04 DEBUG }, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG "985149440" 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG { 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "data": { 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG "result": [ 17:19:04 DEBUG { 17:19:04 DEBUG "metric": { 17:19:04 DEBUG "pod": "ds-cts-0" 17:19:04 DEBUG }, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG "pod": "ds-cts-2" 17:19:04 DEBUG }, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG "0" 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543544" 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543544" 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543544" 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543544" 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543544" 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543544" 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543544" 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543544" 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543544" 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543544" 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543544" 17:19:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543544" 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 DEBUG --- status code --- 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG --- status code --- 17:19:04 INFO [loop_until]: OK (rc = 0) 17:19:04 DEBUG --- status code --- 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG --- stdout --- 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11610m 8719Mi am-6fd554f8f7-btd4s 11705m 8767Mi am-6fd554f8f7-wxcsj 12041m 8775Mi ds-cts-0 9286m 12436Mi ds-cts-1 9312m 12374Mi ds-cts-2 9211m 12419Mi ds-idrepo-0 5672m 3414Mi ds-idrepo-1 5505m 3366Mi ds-idrepo-2 5408m 3424Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7871m 847Mi 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG --- http response --- 17:19:04 INFO [http_cmd]: http status code OK 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:19:04 DEBUG --- stderr --- 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:19:04 DEBUG --- status code --- 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG { 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG { 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:19:04 DEBUG { 17:19:04 DEBUG { 17:19:04 DEBUG { 17:19:04 DEBUG { 17:19:04 DEBUG { 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG { 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:19:04 DEBUG { 17:19:04 DEBUG { 17:19:04 DEBUG --- http response --- 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "data": { 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "data": { 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:19:04 DEBUG "data": { 17:19:04 DEBUG "data": { 17:19:04 DEBUG "data": { 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG "data": { 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG "data": { 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG { 17:19:04 DEBUG "data": { 17:19:04 DEBUG "data": { 17:19:04 DEBUG ----- output ----- 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG "data": { 17:19:04 DEBUG "result": [ 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG "result": [ 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG "result": [ 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG { 17:19:04 DEBUG "result": [ 17:19:04 DEBUG "result": [ 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG { 17:19:04 DEBUG "result": [ 17:19:04 DEBUG { 17:19:04 DEBUG "result": [ 17:19:04 DEBUG { 17:19:04 DEBUG "data": { 17:19:04 DEBUG "result": [ 17:19:04 DEBUG "result": [ 17:19:04 DEBUG "status": "success", 17:19:04 DEBUG { 17:19:04 DEBUG "metric": { 17:19:04 DEBUG "result": [ 17:19:04 DEBUG "metric": { 17:19:04 DEBUG { 17:19:04 DEBUG "metric": { 17:19:04 DEBUG { 17:19:04 DEBUG "metric": { 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG { 17:19:04 DEBUG { 17:19:04 DEBUG "data": { 17:19:04 DEBUG { 17:19:04 DEBUG "instance": "10.142.0.107:9100" 17:19:04 DEBUG { 17:19:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:19:04 DEBUG "metric": { 17:19:04 DEBUG "pod": "ds-cts-0" 17:19:04 DEBUG "metric": { 17:19:04 DEBUG "pod": "ds-idrepo-1" 17:19:04 DEBUG "result": [ 17:19:04 DEBUG "metric": { 17:19:04 DEBUG "metric": { 17:19:04 DEBUG "resultType": "vector", 17:19:04 DEBUG "metric": { 17:19:04 DEBUG }, 17:19:04 DEBUG "metric": { 17:19:04 DEBUG }, 17:19:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:19:04 DEBUG }, 17:19:04 DEBUG "instance": "10.142.0.107:9100" 17:19:04 DEBUG }, 17:19:04 DEBUG { 17:19:04 DEBUG "pod": "ds-idrepo-0" 17:19:04 DEBUG "instance": "10.142.0.107:9100" 17:19:04 DEBUG "result": [ 17:19:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:19:04 DEBUG "value": [ 17:19:04 DEBUG "pod": "ds-idrepo-1" 17:19:04 DEBUG "value": [ 17:19:04 DEBUG }, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG }, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG "metric": { 17:19:04 DEBUG }, 17:19:04 DEBUG }, 17:19:04 DEBUG { 17:19:04 DEBUG }, 17:19:04 DEBUG 17:19:04 DEBUG }, 17:19:04 DEBUG 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 17:19:04 DEBUG "pod": "ds-idrepo-1" 17:19:04 DEBUG "value": [ 17:19:04 DEBUG "value": [ 17:19:04 DEBUG "metric": { 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG }, 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:19:04 DEBUG 17:19:04 DEBUG "value": [ 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG }, 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG "instance": "10.142.0.43:9100" 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG }, 17:19:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:19:04 DEBUG 17:19:04 DEBUG "pod": "ds-cts-2" 17:19:04 DEBUG }, 17:19:04 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG [...] 17:19:04 DEBUG 17:19:04 DEBUG 17:19:04 DEBUG "value": [ 17:19:04 DEBUG }, 17:19:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:19:04 DEBUG 17:19:04 DEBUG "value": [ 17:19:04 DEBUG }, 17:19:04 DEBUG 17:19:04 DEBUG "pod": "ds-idrepo-2" 17:19:04 DEBUG "instance": "10.142.0.43:9100" 17:19:04 DEBUG 17:19:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:19:04 DEBUG 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG }, 17:19:04 DEBUG "instance": "10.142.0.43:9100" 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 17:19:04 DEBUG }, 17:19:04 DEBUG }, 17:19:04 DEBUG 17:19:04 DEBUG }, 17:19:04 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:19:04 DEBUG "56739534" 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG }, 17:19:04 DEBUG "1.3333333333333333" 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:19:04 DEBUG "value": [ 17:19:04 DEBUG "value": [ 17:19:04 DEBUG 17:19:04 DEBUG "value": [ 17:19:04 DEBUG }, 17:19:04 DEBUG ] 17:19:04 DEBUG "2332.8" 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG "0" 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG "value": [ 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG "0.0185" 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG }, 17:19:04 DEBUG ] 17:19:04 DEBUG ] 17:19:04 DEBUG "0" 17:19:04 DEBUG "0.06696666666661884" 17:19:04 DEBUG }, 17:19:04 DEBUG "7.878441213084489" 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG "56746309.86666667" 17:19:04 DEBUG "value": [ 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG ] 17:19:04 DEBUG "value": [ 17:19:04 DEBUG ] 17:19:04 DEBUG "0" 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG 1713543544, 17:19:04 DEBUG "2332.9333333333334" 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG "0.000018859455112094485" 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG ] 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG } 17:19:04 DEBUG } 17:20:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543602" 17:20:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543602" 17:20:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543602" 17:20:02 INFO [http_cmd]: http status code OK 17:20:02 INFO [http_cmd]: http status code OK 17:20:02 DEBUG --- status code --- 17:20:02 DEBUG --- status code --- 17:20:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:02 DEBUG --- http response --- 17:20:02 DEBUG --- http response --- 17:20:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:20:02 ERROR [http_cmd]: ERROR 17:20:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:20:02 ERROR --- status code --- 17:20:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:20:02 ERROR --- http response --- 17:20:02 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:20:02 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:20:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543602" 17:20:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543602" 17:20:02 INFO [http_cmd]: http status code OK 17:20:02 DEBUG --- status code --- 17:20:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:02 DEBUG --- http response --- 17:20:02 INFO [http_cmd]: http status code OK 17:20:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:20:02 DEBUG --- status code --- 17:20:02 DEBUG ----- output ----- 17:20:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:02 DEBUG { 17:20:02 DEBUG --- http response --- 17:20:02 DEBUG "status": "success", 17:20:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:20:02 DEBUG "data": { 17:20:02 DEBUG ----- output ----- 17:20:02 DEBUG "resultType": "vector", 17:20:02 DEBUG { 17:20:02 DEBUG "result": [ 17:20:02 DEBUG "status": "success", 17:20:02 DEBUG { 17:20:02 DEBUG "data": { 17:20:02 DEBUG "metric": { 17:20:02 DEBUG "resultType": "vector", 17:20:02 DEBUG "pod": "ds-idrepo-0" 17:20:02 DEBUG "result": [ 17:20:02 DEBUG }, 17:20:02 DEBUG { 17:20:02 DEBUG "value": [ 17:20:02 DEBUG "metric": { 17:20:02 DEBUG 17:20:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:20:02 DEBUG 17:20:02 DEBUG }, 17:20:02 DEBUG 17:20:02 DEBUG "value": [ 17:20:02 DEBUG [...] 17:20:02 DEBUG 17:20:02 DEBUG 17:20:02 DEBUG 17:20:02 DEBUG 17:20:02 DEBUG 17:20:02 DEBUG 17:20:02 DEBUG [...] 17:20:02 DEBUG "pod": "ds-cts-2" 17:20:02 DEBUG 17:20:02 DEBUG }, 17:20:02 DEBUG 17:20:02 DEBUG "value": [ 17:20:02 DEBUG 17:20:02 DEBUG 1713543602, 17:20:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:20:02 DEBUG "0" 17:20:02 DEBUG }, 17:20:02 DEBUG ] 17:20:02 DEBUG "value": [ 17:20:02 DEBUG } 17:20:02 DEBUG 1713543602, 17:20:02 DEBUG ] 17:20:02 DEBUG "2331.7666666666664" 17:20:02 DEBUG } 17:20:02 DEBUG ] 17:20:02 DEBUG } 17:20:02 DEBUG } 17:20:02 DEBUG ] 17:20:02 DEBUG } 17:20:02 DEBUG } 17:20:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543602" 17:20:02 INFO [http_cmd]: http status code OK 17:20:02 DEBUG --- status code --- 17:20:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:02 DEBUG --- http response --- 17:20:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:20:02 DEBUG ----- output ----- 17:20:02 DEBUG { 17:20:02 DEBUG "status": "success", 17:20:02 DEBUG "data": { 17:20:02 DEBUG "resultType": "vector", 17:20:02 DEBUG "result": [ 17:20:02 DEBUG { 17:20:02 DEBUG "metric": { 17:20:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:20:02 DEBUG }, 17:20:02 DEBUG "value": [ 17:20:02 DEBUG 17:20:02 DEBUG 17:20:02 DEBUG 17:20:02 DEBUG [...] 17:20:02 DEBUG 17:20:02 DEBUG 17:20:02 DEBUG 17:20:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:20:02 DEBUG }, 17:20:02 DEBUG "value": [ 17:20:02 DEBUG 1713543602, 17:20:02 DEBUG "0" 17:20:02 DEBUG ] 17:20:02 DEBUG } 17:20:02 DEBUG ] 17:20:02 DEBUG } 17:20:02 DEBUG } 17:20:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543603" 17:20:03 INFO [http_cmd]: http status code OK 17:20:03 DEBUG --- status code --- 17:20:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:03 DEBUG --- http response --- 17:20:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:20:03 DEBUG ----- output ----- 17:20:03 DEBUG { 17:20:03 DEBUG "status": "success", 17:20:03 DEBUG "data": { 17:20:03 DEBUG "resultType": "vector", 17:20:03 DEBUG "result": [ 17:20:03 DEBUG { 17:20:03 DEBUG "metric": { 17:20:03 DEBUG "pod": "ds-idrepo-0" 17:20:03 DEBUG }, 17:20:03 DEBUG "value": [ 17:20:03 DEBUG 17:20:03 DEBUG 17:20:03 DEBUG 17:20:03 DEBUG [...] 17:20:03 DEBUG 17:20:03 DEBUG 17:20:03 DEBUG 17:20:03 DEBUG "pod": "idm-b85f9d978-txpwv" 17:20:03 DEBUG }, 17:20:03 DEBUG "value": [ 17:20:03 DEBUG 1713543603, 17:20:03 DEBUG "3715.0924848118684" 17:20:03 DEBUG ] 17:20:03 DEBUG } 17:20:03 DEBUG ] 17:20:03 DEBUG } 17:20:03 DEBUG } 17:20:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543603" 17:20:03 INFO [http_cmd]: http status code OK 17:20:03 DEBUG --- status code --- 17:20:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:03 DEBUG --- http response --- 17:20:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:20:03 DEBUG ----- output ----- 17:20:03 DEBUG { 17:20:03 DEBUG "status": "success", 17:20:03 DEBUG "data": { 17:20:03 DEBUG "resultType": "vector", 17:20:03 DEBUG "result": [ 17:20:03 DEBUG { 17:20:03 DEBUG "metric": { 17:20:03 DEBUG "instance": "10.142.0.107:9100" 17:20:03 DEBUG }, 17:20:03 DEBUG "value": [ 17:20:03 DEBUG 17:20:03 DEBUG 17:20:03 DEBUG 17:20:03 DEBUG [...] 17:20:03 DEBUG 17:20:03 DEBUG 17:20:03 DEBUG 17:20:03 DEBUG "instance": "10.142.0.43:9100" 17:20:03 DEBUG }, 17:20:03 DEBUG "value": [ 17:20:03 DEBUG 1713543603, 17:20:03 DEBUG "0.04233333333332705" 17:20:03 DEBUG ] 17:20:03 DEBUG } 17:20:03 DEBUG ] 17:20:03 DEBUG } 17:20:03 DEBUG } 17:20:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543603" 17:20:03 INFO [http_cmd]: http status code OK 17:20:03 DEBUG --- status code --- 17:20:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:03 DEBUG --- http response --- 17:20:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:20:03 DEBUG ----- output ----- 17:20:03 DEBUG { 17:20:03 DEBUG "status": "success", 17:20:03 DEBUG "data": { 17:20:03 DEBUG "resultType": "vector", 17:20:03 DEBUG "result": [ 17:20:03 DEBUG { 17:20:03 DEBUG "metric": { 17:20:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:20:03 DEBUG }, 17:20:03 DEBUG "value": [ 17:20:03 DEBUG 17:20:03 DEBUG 17:20:03 DEBUG 17:20:03 DEBUG [...] 17:20:03 DEBUG 17:20:03 DEBUG 17:20:03 DEBUG 17:20:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:20:03 DEBUG }, 17:20:03 DEBUG "value": [ 17:20:03 DEBUG 1713543603, 17:20:03 DEBUG "2766392" 17:20:03 DEBUG ] 17:20:03 DEBUG } 17:20:03 DEBUG ] 17:20:03 DEBUG } 17:20:03 DEBUG } 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543604" 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG { 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "data": { 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "result": [ 17:20:04 DEBUG { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "instance": "10.142.0.107:9100" 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG [...] 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "instance": "10.142.0.43:9100" 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG "1680331490304" 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 INFO 17:20:04 INFO [loop_until]: kubectl --namespace=xlou top node 17:20:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:20:04 INFO [loop_until]: OK (rc = 0) 17:20:04 DEBUG --- stdout --- 17:20:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6186Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5518m 18% 6975Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 51m 0% 2673Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11366m 38% 12004Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9568m 32% 15446Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5216Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11571m 38% 11640Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12525m 41% 2963Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13153m 44% 10986Mi 9% gke-xlou-cdm-ds-0f601751-j488 6259m 20% 6644Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9545m 31% 17171Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6283m 21% 7173Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9507m 31% 14884Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8120m 51% 4634Mi 7% 17:20:04 DEBUG --- stderr --- 17:20:04 DEBUG 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543604" 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG { 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "data": { 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "result": [ 17:20:04 DEBUG { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "pod": "ds-idrepo-0" 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG [...] 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "pod": "ds-idrepo-2" 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG "0" 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG { 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "data": { 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "result": [ 17:20:04 DEBUG { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "pod": "ds-idrepo-0" 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG [...] 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "pod": "ds-idrepo-2" 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG "0" 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543604" 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG { 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "data": { 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "result": [ 17:20:04 DEBUG { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "pod": "ds-cts-0" 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG [...] 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "pod": "ds-cts-2" 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG "0" 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543604" 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG --- status code --- 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG { 17:20:04 DEBUG { 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "data": { 17:20:04 DEBUG "data": { 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "result": [ 17:20:04 DEBUG "result": [ 17:20:04 DEBUG { 17:20:04 DEBUG { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:20:04 DEBUG "pod": "ds-cts-0" 17:20:04 DEBUG }, 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG [...] 17:20:04 DEBUG [...] 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:20:04 DEBUG "pod": "ds-cts-2" 17:20:04 DEBUG }, 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG "984530944" 17:20:04 DEBUG "0" 17:20:04 DEBUG ] 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG ] 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 INFO 17:20:04 INFO [loop_until]: kubectl --namespace=xlou top pods 17:20:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543604" 17:20:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543604" 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- http response --- 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG --- http response --- 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG --- status code --- 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG --- status code --- 17:20:04 INFO [loop_until]: OK (rc = 0) 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG --- status code --- 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:20:04 DEBUG --- status code --- 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 INFO [http_cmd]: http status code OK 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- stdout --- 17:20:04 DEBUG { 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG --- status code --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11457m 8721Mi am-6fd554f8f7-btd4s 11160m 8768Mi am-6fd554f8f7-wxcsj 12219m 8778Mi ds-cts-0 9325m 12437Mi ds-cts-1 9310m 12375Mi ds-cts-2 9807m 12420Mi ds-idrepo-0 5786m 3414Mi ds-idrepo-1 5548m 3370Mi ds-idrepo-2 5600m 3424Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8098m 849Mi 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG { 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:20:04 DEBUG --- stderr --- 17:20:04 DEBUG "data": { 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:20:04 DEBUG --- http response --- 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:20:04 DEBUG 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:20:04 DEBUG "data": { 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:20:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG { 17:20:04 DEBUG { 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG "result": [ 17:20:04 DEBUG { 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG { 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG ----- output ----- 17:20:04 DEBUG { 17:20:04 DEBUG { 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG { 17:20:04 DEBUG { 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG { 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "result": [ 17:20:04 DEBUG { 17:20:04 DEBUG { 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "data": { 17:20:04 DEBUG "data": { 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "data": { 17:20:04 DEBUG "data": { 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG { 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "status": "success", 17:20:04 DEBUG "data": { 17:20:04 DEBUG "data": { 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "data": { 17:20:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "data": { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "pod": "ds-idrepo-0" 17:20:04 DEBUG "data": { 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "result": [ 17:20:04 DEBUG "result": [ 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG }, 17:20:04 DEBUG "result": [ 17:20:04 DEBUG "result": [ 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG }, 17:20:04 DEBUG "data": { 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "result": [ 17:20:04 DEBUG "result": [ 17:20:04 DEBUG { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "result": [ 17:20:04 DEBUG "value": [ 17:20:04 DEBUG { 17:20:04 DEBUG { 17:20:04 DEBUG "result": [ 17:20:04 DEBUG "value": [ 17:20:04 DEBUG "resultType": "vector", 17:20:04 DEBUG "result": [ 17:20:04 DEBUG { 17:20:04 DEBUG { 17:20:04 DEBUG { 17:20:04 DEBUG "pod": "ds-idrepo-0" 17:20:04 DEBUG { 17:20:04 DEBUG 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG { 17:20:04 DEBUG 17:20:04 DEBUG "result": [ 17:20:04 DEBUG { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG }, 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "pod": "ds-idrepo-0" 17:20:04 DEBUG "instance": "10.142.0.107:9100" 17:20:04 DEBUG }, 17:20:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:20:04 DEBUG 17:20:04 DEBUG { 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:20:04 DEBUG "pod": "ds-idrepo-0" 17:20:04 DEBUG "pod": "ds-cts-0" 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "instance": "10.142.0.107:9100" 17:20:04 DEBUG }, 17:20:04 DEBUG 17:20:04 DEBUG "metric": { 17:20:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:20:04 DEBUG }, 17:20:04 DEBUG }, 17:20:04 DEBUG }, 17:20:04 DEBUG }, 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "value": [ 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG [...] 17:20:04 DEBUG "instance": "10.142.0.107:9100" 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG "value": [ 17:20:04 DEBUG "value": [ 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 17:20:04 DEBUG [...] 17:20:04 DEBUG 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG }, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG [...] 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG [...] 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG [...] 17:20:04 DEBUG "pod": "ds-idrepo-2" 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG [...] 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG [...] 17:20:04 DEBUG [...] 17:20:04 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:20:04 DEBUG }, 17:20:04 DEBUG [...] 17:20:04 DEBUG 17:20:04 DEBUG }, 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG [...] 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "value": [ 17:20:04 DEBUG [...] 17:20:04 DEBUG [...] 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG }, 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG "383.3530916578274" 17:20:04 DEBUG 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG }, 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "value": [ 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:20:04 DEBUG "0" 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:20:04 DEBUG 17:20:04 DEBUG "value": [ 17:20:04 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:20:04 DEBUG "instance": "10.142.0.43:9100" 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG ] 17:20:04 DEBUG 17:20:04 DEBUG }, 17:20:04 DEBUG ] 17:20:04 DEBUG 17:20:04 DEBUG 17:20:04 DEBUG }, 17:20:04 DEBUG "pod": "ds-cts-2" 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG }, 17:20:04 DEBUG }, 17:20:04 DEBUG "2330.9333333333334" 17:20:04 DEBUG } 17:20:04 DEBUG 17:20:04 DEBUG "value": [ 17:20:04 DEBUG } 17:20:04 DEBUG "instance": "10.142.0.43:9100" 17:20:04 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:20:04 DEBUG "value": [ 17:20:04 DEBUG }, 17:20:04 DEBUG "2331.0666666666666" 17:20:04 DEBUG "value": [ 17:20:04 DEBUG "value": [ 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG "instance": "10.142.0.43:9100" 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG ] 17:20:04 DEBUG }, 17:20:04 DEBUG }, 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG "value": [ 17:20:04 DEBUG ] 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG } 17:20:04 DEBUG }, 17:20:04 DEBUG "0.019" 17:20:04 DEBUG } 17:20:04 DEBUG "value": [ 17:20:04 DEBUG "value": [ 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG } 17:20:04 DEBUG "0.002890403693768368" 17:20:04 DEBUG "56533437.3" 17:20:04 DEBUG ] 17:20:04 DEBUG "value": [ 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG "7.849533009700064" 17:20:04 DEBUG "0" 17:20:04 DEBUG "1.9666666666666666" 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG 1713543604, 17:20:04 DEBUG "0.06519999999998352" 17:20:04 DEBUG ] 17:20:04 DEBUG "56541070" 17:20:04 DEBUG ] 17:20:04 DEBUG ] 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG ] 17:20:04 DEBUG ] 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG ] 17:20:04 DEBUG ] 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG ] 17:20:04 DEBUG ] 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:20:04 DEBUG } 17:21:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543662" 17:21:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543662" 17:21:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543662" 17:21:02 INFO [http_cmd]: http status code OK 17:21:02 ERROR [http_cmd]: ERROR 17:21:02 DEBUG --- status code --- 17:21:02 ERROR --- status code --- 17:21:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:21:02 DEBUG --- http response --- 17:21:02 ERROR --- http response --- 17:21:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:21:02 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:21:02 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:21:02 INFO [http_cmd]: http status code OK 17:21:02 DEBUG --- status code --- 17:21:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:02 DEBUG --- http response --- 17:21:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:21:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543662" 17:21:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543662" 17:21:02 INFO [http_cmd]: http status code OK 17:21:02 DEBUG --- status code --- 17:21:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:02 DEBUG --- http response --- 17:21:02 INFO [http_cmd]: http status code OK 17:21:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:21:02 DEBUG --- status code --- 17:21:02 DEBUG ----- output ----- 17:21:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:02 DEBUG { 17:21:02 DEBUG --- http response --- 17:21:02 DEBUG "status": "success", 17:21:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:21:02 DEBUG "data": { 17:21:02 DEBUG ----- output ----- 17:21:02 DEBUG "resultType": "vector", 17:21:02 DEBUG { 17:21:02 DEBUG "result": [ 17:21:02 DEBUG "status": "success", 17:21:02 DEBUG { 17:21:02 DEBUG "data": { 17:21:02 DEBUG "metric": { 17:21:02 DEBUG "resultType": "vector", 17:21:02 DEBUG "pod": "ds-idrepo-0" 17:21:02 DEBUG "result": [ 17:21:02 DEBUG }, 17:21:02 DEBUG { 17:21:02 DEBUG "value": [ 17:21:02 DEBUG "metric": { 17:21:02 DEBUG 17:21:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:21:02 DEBUG 17:21:02 DEBUG }, 17:21:02 DEBUG 17:21:02 DEBUG "value": [ 17:21:02 DEBUG [...] 17:21:02 DEBUG 17:21:02 DEBUG 17:21:02 DEBUG 17:21:02 DEBUG 17:21:02 DEBUG 17:21:02 DEBUG 17:21:02 DEBUG [...] 17:21:02 DEBUG "pod": "ds-cts-2" 17:21:02 DEBUG 17:21:02 DEBUG }, 17:21:02 DEBUG 17:21:02 DEBUG "value": [ 17:21:02 DEBUG 17:21:02 DEBUG 1713543662, 17:21:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:21:02 DEBUG "0" 17:21:02 DEBUG }, 17:21:02 DEBUG ] 17:21:02 DEBUG "value": [ 17:21:02 DEBUG } 17:21:02 DEBUG 1713543662, 17:21:02 DEBUG ] 17:21:02 DEBUG "2330.5333333333333" 17:21:02 DEBUG ] 17:21:02 DEBUG } 17:21:02 DEBUG } 17:21:02 DEBUG ] 17:21:02 DEBUG } 17:21:02 DEBUG } 17:21:02 DEBUG } 17:21:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543663" 17:21:03 INFO [http_cmd]: http status code OK 17:21:03 DEBUG --- status code --- 17:21:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:03 DEBUG --- http response --- 17:21:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:21:03 DEBUG ----- output ----- 17:21:03 DEBUG { 17:21:03 DEBUG "status": "success", 17:21:03 DEBUG "data": { 17:21:03 DEBUG "resultType": "vector", 17:21:03 DEBUG "result": [ 17:21:03 DEBUG { 17:21:03 DEBUG "metric": { 17:21:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:21:03 DEBUG }, 17:21:03 DEBUG "value": [ 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG [...] 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:21:03 DEBUG }, 17:21:03 DEBUG "value": [ 17:21:03 DEBUG 1713543663, 17:21:03 DEBUG "0" 17:21:03 DEBUG ] 17:21:03 DEBUG } 17:21:03 DEBUG ] 17:21:03 DEBUG } 17:21:03 DEBUG } 17:21:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543663" 17:21:03 INFO [http_cmd]: http status code OK 17:21:03 DEBUG --- status code --- 17:21:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:03 DEBUG --- http response --- 17:21:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:21:03 DEBUG ----- output ----- 17:21:03 DEBUG { 17:21:03 DEBUG "status": "success", 17:21:03 DEBUG "data": { 17:21:03 DEBUG "resultType": "vector", 17:21:03 DEBUG "result": [ 17:21:03 DEBUG { 17:21:03 DEBUG "metric": { 17:21:03 DEBUG "pod": "ds-idrepo-0" 17:21:03 DEBUG }, 17:21:03 DEBUG "value": [ 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG [...] 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:21:03 DEBUG }, 17:21:03 DEBUG "value": [ 17:21:03 DEBUG 1713543663, 17:21:03 DEBUG "0" 17:21:03 DEBUG ] 17:21:03 DEBUG } 17:21:03 DEBUG ] 17:21:03 DEBUG } 17:21:03 DEBUG } 17:21:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543663" 17:21:03 INFO [http_cmd]: http status code OK 17:21:03 DEBUG --- status code --- 17:21:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:03 DEBUG --- http response --- 17:21:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:21:03 DEBUG ----- output ----- 17:21:03 DEBUG { 17:21:03 DEBUG "status": "success", 17:21:03 DEBUG "data": { 17:21:03 DEBUG "resultType": "vector", 17:21:03 DEBUG "result": [ 17:21:03 DEBUG { 17:21:03 DEBUG "metric": { 17:21:03 DEBUG "instance": "10.142.0.107:9100" 17:21:03 DEBUG }, 17:21:03 DEBUG "value": [ 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG [...] 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG "instance": "10.142.0.43:9100" 17:21:03 DEBUG }, 17:21:03 DEBUG "value": [ 17:21:03 DEBUG 1713543663, 17:21:03 DEBUG "0.04299999999999879" 17:21:03 DEBUG ] 17:21:03 DEBUG } 17:21:03 DEBUG ] 17:21:03 DEBUG } 17:21:03 DEBUG } 17:21:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543663" 17:21:03 INFO [http_cmd]: http status code OK 17:21:03 DEBUG --- status code --- 17:21:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:03 DEBUG --- http response --- 17:21:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:21:03 DEBUG ----- output ----- 17:21:03 DEBUG { 17:21:03 DEBUG "status": "success", 17:21:03 DEBUG "data": { 17:21:03 DEBUG "resultType": "vector", 17:21:03 DEBUG "result": [ 17:21:03 DEBUG { 17:21:03 DEBUG "metric": { 17:21:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:21:03 DEBUG }, 17:21:03 DEBUG "value": [ 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG [...] 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG 17:21:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:21:03 DEBUG }, 17:21:03 DEBUG "value": [ 17:21:03 DEBUG 1713543663, 17:21:03 DEBUG "2906160.5" 17:21:03 DEBUG ] 17:21:03 DEBUG } 17:21:03 DEBUG ] 17:21:03 DEBUG } 17:21:03 DEBUG } 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543664" 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG { 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG "data": { 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG "result": [ 17:21:04 DEBUG { 17:21:04 DEBUG "metric": { 17:21:04 DEBUG "instance": "10.142.0.107:9100" 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG [...] 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG "instance": "10.142.0.43:9100" 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG "1683159694336" 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 INFO 17:21:04 INFO [loop_until]: kubectl --namespace=xlou top node 17:21:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:21:04 INFO [loop_until]: OK (rc = 0) 17:21:04 DEBUG --- stdout --- 17:21:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6182Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5408m 18% 6991Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 48m 0% 2686Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 10851m 36% 12006Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 8935m 29% 15458Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5227Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11529m 38% 11639Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12299m 41% 2958Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13011m 43% 10991Mi 9% gke-xlou-cdm-ds-0f601751-j488 6279m 21% 6650Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9576m 32% 17181Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6213m 20% 7184Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9596m 32% 14887Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8058m 50% 4635Mi 7% 17:21:04 DEBUG --- stderr --- 17:21:04 DEBUG 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543664" 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543664" 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543664" 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG { 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG { 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG "data": { 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG "data": { 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:21:04 DEBUG "result": [ 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG { 17:21:04 DEBUG "result": [ 17:21:04 DEBUG { 17:21:04 DEBUG "metric": { 17:21:04 DEBUG { 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG "pod": "ds-idrepo-0" 17:21:04 DEBUG "metric": { 17:21:04 DEBUG "data": { 17:21:04 DEBUG }, 17:21:04 DEBUG "pod": "ds-idrepo-0" 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 17:21:04 DEBUG "result": [ 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG { 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG "metric": { 17:21:04 DEBUG 17:21:04 DEBUG [...] 17:21:04 DEBUG "pod": "ds-cts-0" 17:21:04 DEBUG [...] 17:21:04 DEBUG 17:21:04 DEBUG }, 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG "pod": "ds-idrepo-2" 17:21:04 DEBUG 17:21:04 DEBUG "pod": "ds-idrepo-2" 17:21:04 DEBUG }, 17:21:04 DEBUG 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG [...] 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG "0" 17:21:04 DEBUG 17:21:04 DEBUG "0" 17:21:04 DEBUG ] 17:21:04 DEBUG 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG "pod": "ds-cts-2" 17:21:04 DEBUG } 17:21:04 DEBUG ] 17:21:04 DEBUG }, 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG "value": [ 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG } 17:21:04 DEBUG "0" 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543664" 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543664" 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG { 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG "data": { 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG "result": [ 17:21:04 DEBUG { 17:21:04 DEBUG "metric": { 17:21:04 DEBUG "pod": "ds-cts-0" 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG [...] 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG "pod": "ds-cts-2" 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG "0" 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG { 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG "data": { 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG "result": [ 17:21:04 DEBUG { 17:21:04 DEBUG "metric": { 17:21:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG [...] 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG "989143040" 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 INFO 17:21:04 INFO [loop_until]: kubectl --namespace=xlou top pods 17:21:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543664" 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543664" 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG --- http response --- 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543664" 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543664" 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543664" 17:21:04 DEBUG ----- output ----- 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543664" 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543664" 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543664" 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543664" 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543664" 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543664" 17:21:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543664" 17:21:04 DEBUG { 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG "status": "success", 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG --- status code --- 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG "data": { 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG --- status code --- 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG "result": [ 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:21:04 DEBUG { 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 INFO [http_cmd]: http status code OK 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG "metric": { 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG --- status code --- 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG { 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG { 17:21:04 DEBUG "pod": "ds-idrepo-0" 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG { 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG { 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG { 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG }, 17:21:04 DEBUG { 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG --- http response --- 17:21:04 DEBUG "data": { 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:21:04 DEBUG { 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG { 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG "data": { 17:21:04 DEBUG "value": [ 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG "data": { 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:21:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:21:04 INFO [loop_until]: OK (rc = 0) 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG { 17:21:04 DEBUG "data": { 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG "data": { 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG 17:21:04 DEBUG "data": { 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG ----- output ----- 17:21:04 DEBUG --- stdout --- 17:21:04 DEBUG "result": [ 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG "data": { 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG "result": [ 17:21:04 DEBUG 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG "result": [ 17:21:04 DEBUG { 17:21:04 DEBUG { 17:21:04 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11219m 8724Mi am-6fd554f8f7-btd4s 11590m 8769Mi am-6fd554f8f7-wxcsj 11890m 8779Mi ds-cts-0 9182m 12437Mi ds-cts-1 9471m 12375Mi ds-cts-2 9622m 12421Mi ds-idrepo-0 5682m 3414Mi ds-idrepo-1 5534m 3372Mi ds-idrepo-2 5592m 3424Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8244m 850Mi 17:21:04 DEBUG { 17:21:04 DEBUG "metric": { 17:21:04 DEBUG "data": { 17:21:04 DEBUG "result": [ 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG "result": [ 17:21:04 DEBUG { 17:21:04 DEBUG 17:21:04 DEBUG "result": [ 17:21:04 DEBUG { 17:21:04 DEBUG "metric": { 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG --- stderr --- 17:21:04 DEBUG "data": { 17:21:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG { 17:21:04 DEBUG "result": [ 17:21:04 DEBUG { 17:21:04 DEBUG "metric": { 17:21:04 DEBUG [...] 17:21:04 DEBUG { 17:21:04 DEBUG "status": "success", 17:21:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:21:04 DEBUG }, 17:21:04 DEBUG 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG }, 17:21:04 DEBUG "result": [ 17:21:04 DEBUG "metric": { 17:21:04 DEBUG { 17:21:04 DEBUG "metric": { 17:21:04 DEBUG "pod": "ds-idrepo-0" 17:21:04 DEBUG 17:21:04 DEBUG "metric": { 17:21:04 DEBUG "data": { 17:21:04 DEBUG "data": { 17:21:04 DEBUG "value": [ 17:21:04 DEBUG "result": [ 17:21:04 DEBUG "value": [ 17:21:04 DEBUG { 17:21:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:21:04 DEBUG "metric": { 17:21:04 DEBUG "instance": "10.142.0.107:9100" 17:21:04 DEBUG }, 17:21:04 DEBUG 17:21:04 DEBUG "pod": "ds-cts-0" 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG "resultType": "vector", 17:21:04 DEBUG 17:21:04 DEBUG { 17:21:04 DEBUG 17:21:04 DEBUG "metric": { 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 17:21:04 DEBUG }, 17:21:04 DEBUG "result": [ 17:21:04 DEBUG "result": [ 17:21:04 DEBUG 17:21:04 DEBUG "metric": { 17:21:04 DEBUG 17:21:04 DEBUG "instance": "10.142.0.107:9100" 17:21:04 DEBUG "instance": "10.142.0.107:9100" 17:21:04 DEBUG 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 17:21:04 DEBUG "pod": "ds-idrepo-2" 17:21:04 DEBUG "value": [ 17:21:04 DEBUG { 17:21:04 DEBUG { 17:21:04 DEBUG 17:21:04 DEBUG "pod": "ds-idrepo-0" 17:21:04 DEBUG 17:21:04 DEBUG }, 17:21:04 DEBUG }, 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG }, 17:21:04 DEBUG 17:21:04 DEBUG "metric": { 17:21:04 DEBUG "metric": { 17:21:04 DEBUG [...] 17:21:04 DEBUG 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 17:21:04 DEBUG "pod": "ds-idrepo-0" 17:21:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:21:04 DEBUG [...] 17:21:04 DEBUG 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG [...] 17:21:04 DEBUG 17:21:04 DEBUG [...] 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG 17:21:04 DEBUG }, 17:21:04 DEBUG }, 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG [...] 17:21:04 DEBUG 17:21:04 DEBUG "0" 17:21:04 DEBUG [...] 17:21:04 DEBUG "value": [ 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 17:21:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG ] 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG }, 17:21:04 DEBUG 17:21:04 DEBUG [...] 17:21:04 DEBUG [...] 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG } 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:21:04 DEBUG "value": [ 17:21:04 DEBUG [...] 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:21:04 DEBUG 17:21:04 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:21:04 DEBUG ] 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG }, 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG }, 17:21:04 DEBUG "instance": "10.142.0.43:9100" 17:21:04 DEBUG }, 17:21:04 DEBUG } 17:21:04 DEBUG "pod": "ds-cts-2" 17:21:04 DEBUG [...] 17:21:04 DEBUG [...] 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG "0.0205" 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG "value": [ 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG } 17:21:04 DEBUG }, 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG "2330.9666666666667" 17:21:04 DEBUG ] 17:21:04 DEBUG "instance": "10.142.0.43:9100" 17:21:04 DEBUG "instance": "10.142.0.43:9100" 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG }, 17:21:04 DEBUG }, 17:21:04 DEBUG "2330.0333333333333" 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG "0" 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG "2.0666666666666664" 17:21:04 DEBUG 17:21:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:21:04 DEBUG } 17:21:04 DEBUG ] 17:21:04 DEBUG "value": [ 17:21:04 DEBUG "value": [ 17:21:04 DEBUG ] 17:21:04 DEBUG "56566525.33333333" 17:21:04 DEBUG ] 17:21:04 DEBUG ] 17:21:04 DEBUG ] 17:21:04 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:21:04 DEBUG }, 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG } 17:21:04 DEBUG 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG }, 17:21:04 DEBUG "value": [ 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG } 17:21:04 DEBUG "56573427.13333333" 17:21:04 DEBUG ] 17:21:04 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:21:04 DEBUG ] 17:21:04 DEBUG ] 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG } 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG }, 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG "value": [ 17:21:04 DEBUG "0.06679999999995895" 17:21:04 DEBUG "7.829009315555742" 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG "value": [ 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG "0.00003281209946168085" 17:21:04 DEBUG ] 17:21:04 DEBUG ] 17:21:04 DEBUG 1713543664, 17:21:04 DEBUG "0" 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG ] 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG ] 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG ] 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG } 17:21:04 DEBUG } 17:22:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543722" 17:22:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543722" 17:22:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543722" 17:22:02 INFO [http_cmd]: http status code OK 17:22:02 DEBUG --- status code --- 17:22:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:02 DEBUG --- http response --- 17:22:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:22:02 ERROR [http_cmd]: ERROR 17:22:02 INFO [http_cmd]: http status code OK 17:22:02 ERROR --- status code --- 17:22:02 DEBUG --- status code --- 17:22:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:22:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:02 ERROR --- http response --- 17:22:02 DEBUG --- http response --- 17:22:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:22:02 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:22:02 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:22:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543722" 17:22:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543722" 17:22:02 INFO [http_cmd]: http status code OK 17:22:02 DEBUG --- status code --- 17:22:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:02 DEBUG --- http response --- 17:22:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:22:02 DEBUG ----- output ----- 17:22:02 DEBUG { 17:22:02 DEBUG "status": "success", 17:22:02 DEBUG "data": { 17:22:02 DEBUG "resultType": "vector", 17:22:02 DEBUG "result": [ 17:22:02 DEBUG { 17:22:02 DEBUG "metric": { 17:22:02 DEBUG "pod": "ds-idrepo-0" 17:22:02 DEBUG }, 17:22:02 DEBUG "value": [ 17:22:02 DEBUG 17:22:02 DEBUG 17:22:02 DEBUG 17:22:02 DEBUG [...] 17:22:02 DEBUG 17:22:02 DEBUG 17:22:02 DEBUG 17:22:02 DEBUG "pod": "ds-cts-2" 17:22:02 DEBUG }, 17:22:02 DEBUG "value": [ 17:22:02 DEBUG 1713543722, 17:22:02 DEBUG "0" 17:22:02 DEBUG ] 17:22:02 DEBUG } 17:22:02 DEBUG ] 17:22:02 DEBUG } 17:22:02 DEBUG } 17:22:02 INFO [http_cmd]: http status code OK 17:22:02 DEBUG --- status code --- 17:22:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:02 DEBUG --- http response --- 17:22:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:22:02 DEBUG ----- output ----- 17:22:02 DEBUG { 17:22:02 DEBUG "status": "success", 17:22:02 DEBUG "data": { 17:22:02 DEBUG "resultType": "vector", 17:22:02 DEBUG "result": [ 17:22:02 DEBUG { 17:22:02 DEBUG "metric": { 17:22:02 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:22:02 DEBUG }, 17:22:02 DEBUG "value": [ 17:22:02 DEBUG 17:22:02 DEBUG 17:22:02 DEBUG 17:22:02 DEBUG [...] 17:22:02 DEBUG 17:22:02 DEBUG 17:22:02 DEBUG 17:22:02 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:22:02 DEBUG }, 17:22:02 DEBUG "value": [ 17:22:02 DEBUG 1713543722, 17:22:02 DEBUG "2341.5" 17:22:02 DEBUG ] 17:22:02 DEBUG } 17:22:02 DEBUG ] 17:22:02 DEBUG } 17:22:02 DEBUG } 17:22:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543723" 17:22:03 INFO [http_cmd]: http status code OK 17:22:03 DEBUG --- status code --- 17:22:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:03 DEBUG --- http response --- 17:22:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:22:03 DEBUG ----- output ----- 17:22:03 DEBUG { 17:22:03 DEBUG "status": "success", 17:22:03 DEBUG "data": { 17:22:03 DEBUG "resultType": "vector", 17:22:03 DEBUG "result": [ 17:22:03 DEBUG { 17:22:03 DEBUG "metric": { 17:22:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:22:03 DEBUG }, 17:22:03 DEBUG "value": [ 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG [...] 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:22:03 DEBUG }, 17:22:03 DEBUG "value": [ 17:22:03 DEBUG 1713543723, 17:22:03 DEBUG "0" 17:22:03 DEBUG ] 17:22:03 DEBUG } 17:22:03 DEBUG ] 17:22:03 DEBUG } 17:22:03 DEBUG } 17:22:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543723" 17:22:03 INFO [http_cmd]: http status code OK 17:22:03 DEBUG --- status code --- 17:22:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:03 DEBUG --- http response --- 17:22:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 17:22:03 DEBUG ----- output ----- 17:22:03 DEBUG { 17:22:03 DEBUG "status": "success", 17:22:03 DEBUG "data": { 17:22:03 DEBUG "resultType": "vector", 17:22:03 DEBUG "result": [ 17:22:03 DEBUG { 17:22:03 DEBUG "metric": { 17:22:03 DEBUG "pod": "ds-idrepo-0" 17:22:03 DEBUG }, 17:22:03 DEBUG "value": [ 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG [...] 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG "pod": "ds-idrepo-2" 17:22:03 DEBUG }, 17:22:03 DEBUG "value": [ 17:22:03 DEBUG 1713543723, 17:22:03 DEBUG "11239.287975796806" 17:22:03 DEBUG ] 17:22:03 DEBUG } 17:22:03 DEBUG ] 17:22:03 DEBUG } 17:22:03 DEBUG } 17:22:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543723" 17:22:03 INFO [http_cmd]: http status code OK 17:22:03 DEBUG --- status code --- 17:22:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:03 DEBUG --- http response --- 17:22:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:22:03 DEBUG ----- output ----- 17:22:03 DEBUG { 17:22:03 DEBUG "status": "success", 17:22:03 DEBUG "data": { 17:22:03 DEBUG "resultType": "vector", 17:22:03 DEBUG "result": [ 17:22:03 DEBUG { 17:22:03 DEBUG "metric": { 17:22:03 DEBUG "instance": "10.142.0.107:9100" 17:22:03 DEBUG }, 17:22:03 DEBUG "value": [ 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG [...] 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG "instance": "10.142.0.43:9100" 17:22:03 DEBUG }, 17:22:03 DEBUG "value": [ 17:22:03 DEBUG 1713543723, 17:22:03 DEBUG "0.04099999999999302" 17:22:03 DEBUG ] 17:22:03 DEBUG } 17:22:03 DEBUG ] 17:22:03 DEBUG } 17:22:03 DEBUG } 17:22:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543723" 17:22:03 INFO [http_cmd]: http status code OK 17:22:03 DEBUG --- status code --- 17:22:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:03 DEBUG --- http response --- 17:22:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:22:03 DEBUG ----- output ----- 17:22:03 DEBUG { 17:22:03 DEBUG "status": "success", 17:22:03 DEBUG "data": { 17:22:03 DEBUG "resultType": "vector", 17:22:03 DEBUG "result": [ 17:22:03 DEBUG { 17:22:03 DEBUG "metric": { 17:22:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:22:03 DEBUG }, 17:22:03 DEBUG "value": [ 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG [...] 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG 17:22:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:22:03 DEBUG }, 17:22:03 DEBUG "value": [ 17:22:03 DEBUG 1713543723, 17:22:03 DEBUG "3046060" 17:22:03 DEBUG ] 17:22:03 DEBUG } 17:22:03 DEBUG ] 17:22:03 DEBUG } 17:22:03 DEBUG } 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543724" 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG { 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "result": [ 17:22:04 DEBUG { 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "instance": "10.142.0.107:9100" 17:22:04 DEBUG }, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG "instance": "10.142.0.43:9100" 17:22:04 DEBUG }, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG "1685902320640" 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 INFO 17:22:04 INFO [loop_until]: kubectl --namespace=xlou top node 17:22:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543724" 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG { 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "result": [ 17:22:04 DEBUG { 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:22:04 DEBUG }, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:22:04 DEBUG }, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG "991080448" 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG { 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "result": [ 17:22:04 DEBUG { 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "pod": "ds-idrepo-0" 17:22:04 DEBUG }, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG "pod": "ds-idrepo-2" 17:22:04 DEBUG }, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG "0" 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG --- http response --- 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG { 17:22:04 DEBUG { 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "data": { 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "result": [ 17:22:04 DEBUG "result": [ 17:22:04 DEBUG { 17:22:04 DEBUG { 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "pod": "ds-cts-0" 17:22:04 DEBUG "pod": "ds-idrepo-0" 17:22:04 DEBUG }, 17:22:04 DEBUG }, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG "pod": "ds-cts-2" 17:22:04 DEBUG "pod": "ds-idrepo-2" 17:22:04 DEBUG }, 17:22:04 DEBUG }, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG "0" 17:22:04 DEBUG "0" 17:22:04 DEBUG ] 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543724" 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG { 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "result": [ 17:22:04 DEBUG { 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "pod": "ds-cts-0" 17:22:04 DEBUG }, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG "pod": "ds-cts-2" 17:22:04 DEBUG }, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG "0" 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 INFO 17:22:04 INFO [loop_until]: kubectl --namespace=xlou top pods 17:22:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543724" 17:22:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543724" 17:22:04 INFO [loop_until]: OK (rc = 0) 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG --- stdout --- 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG --- status code --- 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG --- status code --- 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6178Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5400m 18% 6990Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 50m 0% 2690Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11250m 37% 11992Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9490m 31% 15453Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 136m 0% 5225Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11782m 39% 11648Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12346m 41% 2956Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13115m 43% 10994Mi 9% gke-xlou-cdm-ds-0f601751-j488 6279m 21% 6650Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9675m 32% 17173Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6228m 20% 7186Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9666m 32% 14890Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8082m 50% 4642Mi 7% 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG --- status code --- 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG --- stderr --- 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG --- status code --- 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG --- http response --- 17:22:04 INFO [http_cmd]: http status code OK 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG --- status code --- 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG --- http response --- 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG { 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG { 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG { 17:22:04 DEBUG { 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:22:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:22:04 DEBUG { 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG { 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG { 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG { 17:22:04 DEBUG { 17:22:04 DEBUG { 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG ----- output ----- 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "data": { 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG { 17:22:04 DEBUG { 17:22:04 DEBUG "data": { 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "status": "success", 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "result": [ 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "result": [ 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "result": [ 17:22:04 DEBUG "result": [ 17:22:04 DEBUG "data": { 17:22:04 DEBUG "result": [ 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "data": { 17:22:04 DEBUG "data": { 17:22:04 DEBUG "result": [ 17:22:04 DEBUG { 17:22:04 DEBUG "result": [ 17:22:04 DEBUG { 17:22:04 DEBUG "result": [ 17:22:04 DEBUG { 17:22:04 DEBUG { 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG { 17:22:04 DEBUG "result": [ 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG "resultType": "vector", 17:22:04 DEBUG { 17:22:04 DEBUG "metric": { 17:22:04 DEBUG { 17:22:04 DEBUG "metric": { 17:22:04 DEBUG { 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "result": [ 17:22:04 DEBUG "metric": { 17:22:04 DEBUG { 17:22:04 DEBUG "result": [ 17:22:04 DEBUG "result": [ 17:22:04 DEBUG "pod": "ds-cts-0" 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "pod": "ds-idrepo-0" 17:22:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:22:04 DEBUG }, 17:22:04 DEBUG "instance": "10.142.0.107:9100" 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:22:04 DEBUG { 17:22:04 DEBUG }, 17:22:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:22:04 DEBUG }, 17:22:04 DEBUG "pod": "ds-idrepo-0" 17:22:04 DEBUG }, 17:22:04 DEBUG "pod": "ds-idrepo-0" 17:22:04 DEBUG "value": [ 17:22:04 DEBUG { 17:22:04 DEBUG }, 17:22:04 DEBUG }, 17:22:04 DEBUG { 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG "value": [ 17:22:04 DEBUG }, 17:22:04 DEBUG 17:22:04 DEBUG "metric": { 17:22:04 DEBUG "value": [ 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG "metric": { 17:22:04 DEBUG 17:22:04 DEBUG }, 17:22:04 DEBUG 17:22:04 DEBUG }, 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG "value": [ 17:22:04 DEBUG "instance": "10.142.0.107:9100" 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG "pod": "ds-idrepo-0" 17:22:04 DEBUG "instance": "10.142.0.107:9100" 17:22:04 DEBUG 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG }, 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG }, 17:22:04 DEBUG }, 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG "value": [ 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG [...] 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG "pod": "ds-cts-2" 17:22:04 DEBUG 17:22:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:22:04 DEBUG 17:22:04 DEBUG "pod": "ds-idrepo-2" 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:22:04 DEBUG [...] 17:22:04 DEBUG [...] 17:22:04 DEBUG }, 17:22:04 DEBUG 17:22:04 DEBUG }, 17:22:04 DEBUG 17:22:04 DEBUG }, 17:22:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:22:04 DEBUG [...] 17:22:04 DEBUG "pod": "ds-idrepo-2" 17:22:04 DEBUG "instance": "10.142.0.43:9100" 17:22:04 DEBUG }, 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG "value": [ 17:22:04 DEBUG }, 17:22:04 DEBUG 17:22:04 DEBUG }, 17:22:04 DEBUG }, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG "pod": "ds-idrepo-2" 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 17:22:04 DEBUG "value": [ 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG 17:22:04 DEBUG 17:22:04 DEBUG "3.1666666666666665" 17:22:04 DEBUG }, 17:22:04 DEBUG "2341.8" 17:22:04 DEBUG }, 17:22:04 DEBUG "0" 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG "7.858965028148104" 17:22:04 DEBUG "pod": "ds-idrepo-2" 17:22:04 DEBUG "instance": "10.142.0.43:9100" 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG "value": [ 17:22:04 DEBUG ] 17:22:04 DEBUG "2341.5333333333333" 17:22:04 DEBUG "instance": "10.142.0.43:9100" 17:22:04 DEBUG "4610.989938788433" 17:22:04 DEBUG "56890146.833333336" 17:22:04 DEBUG ] 17:22:04 DEBUG }, 17:22:04 DEBUG }, 17:22:04 DEBUG "value": [ 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG } 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG }, 17:22:04 DEBUG ] 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG "value": [ 17:22:04 DEBUG "value": [ 17:22:04 DEBUG "0.0215" 17:22:04 DEBUG ] 17:22:04 DEBUG ] 17:22:04 DEBUG "0" 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG "value": [ 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG 1713543724, 17:22:04 DEBUG ] 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG "5.501010659255818" 17:22:04 DEBUG "0.06956666666666346" 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG "56883468" 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG ] 17:22:04 DEBUG ] 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG ] 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 DEBUG } 17:22:04 INFO [loop_until]: OK (rc = 0) 17:22:04 DEBUG --- stdout --- 17:22:04 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11346m 8724Mi am-6fd554f8f7-btd4s 11301m 8772Mi am-6fd554f8f7-wxcsj 11958m 8781Mi ds-cts-0 9106m 12438Mi ds-cts-1 9427m 12376Mi ds-cts-2 9659m 12421Mi ds-idrepo-0 5522m 3420Mi ds-idrepo-1 5541m 3372Mi ds-idrepo-2 5411m 3424Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7916m 853Mi 17:22:04 DEBUG --- stderr --- 17:22:04 DEBUG 17:23:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543782" 17:23:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543782" 17:23:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543782" 17:23:02 INFO [http_cmd]: http status code OK 17:23:02 DEBUG --- status code --- 17:23:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:02 DEBUG --- http response --- 17:23:02 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:23:02 INFO [http_cmd]: http status code OK 17:23:02 ERROR [http_cmd]: ERROR 17:23:02 DEBUG --- status code --- 17:23:02 ERROR --- status code --- 17:23:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:02 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:23:03 DEBUG --- http response --- 17:23:03 ERROR --- http response --- 17:23:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:23:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:23:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:23:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543783" 17:23:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543783" 17:23:03 INFO [http_cmd]: http status code OK 17:23:03 DEBUG --- status code --- 17:23:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:03 DEBUG --- http response --- 17:23:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:23:03 DEBUG ----- output ----- 17:23:03 DEBUG { 17:23:03 DEBUG "status": "success", 17:23:03 DEBUG "data": { 17:23:03 DEBUG "resultType": "vector", 17:23:03 DEBUG "result": [ 17:23:03 DEBUG { 17:23:03 DEBUG "metric": { 17:23:03 DEBUG "pod": "ds-idrepo-0" 17:23:03 DEBUG }, 17:23:03 DEBUG "value": [ 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG [...] 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG "pod": "ds-cts-2" 17:23:03 DEBUG }, 17:23:03 DEBUG "value": [ 17:23:03 DEBUG 1713543783, 17:23:03 DEBUG "0" 17:23:03 DEBUG ] 17:23:03 DEBUG } 17:23:03 DEBUG ] 17:23:03 DEBUG } 17:23:03 DEBUG } 17:23:03 INFO [http_cmd]: http status code OK 17:23:03 DEBUG --- status code --- 17:23:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:03 DEBUG --- http response --- 17:23:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:23:03 DEBUG ----- output ----- 17:23:03 DEBUG { 17:23:03 DEBUG "status": "success", 17:23:03 DEBUG "data": { 17:23:03 DEBUG "resultType": "vector", 17:23:03 DEBUG "result": [ 17:23:03 DEBUG { 17:23:03 DEBUG "metric": { 17:23:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:23:03 DEBUG }, 17:23:03 DEBUG "value": [ 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG [...] 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:23:03 DEBUG }, 17:23:03 DEBUG "value": [ 17:23:03 DEBUG 1713543783, 17:23:03 DEBUG "2332.8" 17:23:03 DEBUG ] 17:23:03 DEBUG } 17:23:03 DEBUG ] 17:23:03 DEBUG } 17:23:03 DEBUG } 17:23:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543783" 17:23:03 INFO [http_cmd]: http status code OK 17:23:03 DEBUG --- status code --- 17:23:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:03 DEBUG --- http response --- 17:23:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:23:03 DEBUG ----- output ----- 17:23:03 DEBUG { 17:23:03 DEBUG "status": "success", 17:23:03 DEBUG "data": { 17:23:03 DEBUG "resultType": "vector", 17:23:03 DEBUG "result": [ 17:23:03 DEBUG { 17:23:03 DEBUG "metric": { 17:23:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:23:03 DEBUG }, 17:23:03 DEBUG "value": [ 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG [...] 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:23:03 DEBUG }, 17:23:03 DEBUG "value": [ 17:23:03 DEBUG 1713543783, 17:23:03 DEBUG "0" 17:23:03 DEBUG ] 17:23:03 DEBUG } 17:23:03 DEBUG ] 17:23:03 DEBUG } 17:23:03 DEBUG } 17:23:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543783" 17:23:03 INFO [http_cmd]: http status code OK 17:23:03 DEBUG --- status code --- 17:23:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:03 DEBUG --- http response --- 17:23:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:23:03 DEBUG ----- output ----- 17:23:03 DEBUG { 17:23:03 DEBUG "status": "success", 17:23:03 DEBUG "data": { 17:23:03 DEBUG "resultType": "vector", 17:23:03 DEBUG "result": [ 17:23:03 DEBUG { 17:23:03 DEBUG "metric": { 17:23:03 DEBUG "pod": "ds-idrepo-1" 17:23:03 DEBUG }, 17:23:03 DEBUG "value": [ 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG [...] 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG "pod": "am-6fd554f8f7-btd4s" 17:23:03 DEBUG }, 17:23:03 DEBUG "value": [ 17:23:03 DEBUG 1713543783, 17:23:03 DEBUG "0" 17:23:03 DEBUG ] 17:23:03 DEBUG } 17:23:03 DEBUG ] 17:23:03 DEBUG } 17:23:03 DEBUG } 17:23:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543783" 17:23:03 INFO [http_cmd]: http status code OK 17:23:03 DEBUG --- status code --- 17:23:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:03 DEBUG --- http response --- 17:23:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:23:03 DEBUG ----- output ----- 17:23:03 DEBUG { 17:23:03 DEBUG "status": "success", 17:23:03 DEBUG "data": { 17:23:03 DEBUG "resultType": "vector", 17:23:03 DEBUG "result": [ 17:23:03 DEBUG { 17:23:03 DEBUG "metric": { 17:23:03 DEBUG "instance": "10.142.0.107:9100" 17:23:03 DEBUG }, 17:23:03 DEBUG "value": [ 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG [...] 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG "instance": "10.142.0.43:9100" 17:23:03 DEBUG }, 17:23:03 DEBUG "value": [ 17:23:03 DEBUG 1713543783, 17:23:03 DEBUG "0.03999999999999961" 17:23:03 DEBUG ] 17:23:03 DEBUG } 17:23:03 DEBUG ] 17:23:03 DEBUG } 17:23:03 DEBUG } 17:23:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543783" 17:23:03 INFO [http_cmd]: http status code OK 17:23:03 DEBUG --- status code --- 17:23:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:03 DEBUG --- http response --- 17:23:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:23:03 DEBUG ----- output ----- 17:23:03 DEBUG { 17:23:03 DEBUG "status": "success", 17:23:03 DEBUG "data": { 17:23:03 DEBUG "resultType": "vector", 17:23:03 DEBUG "result": [ 17:23:03 DEBUG { 17:23:03 DEBUG "metric": { 17:23:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:23:03 DEBUG }, 17:23:03 DEBUG "value": [ 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG [...] 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG 17:23:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:23:03 DEBUG }, 17:23:03 DEBUG "value": [ 17:23:03 DEBUG 1713543783, 17:23:03 DEBUG "3186177.5" 17:23:03 DEBUG ] 17:23:03 DEBUG } 17:23:03 DEBUG ] 17:23:03 DEBUG } 17:23:03 DEBUG } 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543784" 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG { 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "data": { 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "result": [ 17:23:04 DEBUG { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "instance": "10.142.0.107:9100" 17:23:04 DEBUG }, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG "instance": "10.142.0.43:9100" 17:23:04 DEBUG }, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG "1688736175104" 17:23:04 DEBUG ] 17:23:04 DEBUG } 17:23:04 DEBUG ] 17:23:04 DEBUG } 17:23:04 DEBUG } 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543784" 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG { 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "data": { 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "result": [ 17:23:04 DEBUG { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "pod": "ds-idrepo-0" 17:23:04 DEBUG }, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG "pod": "ds-idrepo-2" 17:23:04 DEBUG }, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG "0" 17:23:04 DEBUG ] 17:23:04 DEBUG } 17:23:04 DEBUG ] 17:23:04 DEBUG } 17:23:04 DEBUG } 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG { 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "data": { 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "result": [ 17:23:04 DEBUG { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:23:04 DEBUG }, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:23:04 DEBUG }, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG "992354304" 17:23:04 DEBUG ] 17:23:04 DEBUG } 17:23:04 DEBUG ] 17:23:04 DEBUG } 17:23:04 DEBUG } 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG { 17:23:04 DEBUG { 17:23:04 DEBUG { 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "data": { 17:23:04 DEBUG "data": { 17:23:04 DEBUG "data": { 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "result": [ 17:23:04 DEBUG "result": [ 17:23:04 DEBUG "result": [ 17:23:04 DEBUG { 17:23:04 DEBUG { 17:23:04 DEBUG { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "pod": "ds-idrepo-0" 17:23:04 DEBUG "pod": "ds-cts-0" 17:23:04 DEBUG "pod": "ds-cts-0" 17:23:04 DEBUG }, 17:23:04 DEBUG }, 17:23:04 DEBUG }, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG "value": [ 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG [...] 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG "pod": "ds-idrepo-2" 17:23:04 DEBUG "pod": "ds-cts-2" 17:23:04 DEBUG "pod": "ds-cts-2" 17:23:04 DEBUG }, 17:23:04 DEBUG }, 17:23:04 DEBUG }, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG "value": [ 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG "0" 17:23:04 DEBUG "0" 17:23:04 DEBUG "0" 17:23:04 DEBUG ] 17:23:04 DEBUG ] 17:23:04 DEBUG ] 17:23:04 DEBUG } 17:23:04 DEBUG } 17:23:04 DEBUG } 17:23:04 DEBUG ] 17:23:04 DEBUG ] 17:23:04 DEBUG ] 17:23:04 DEBUG } 17:23:04 DEBUG } 17:23:04 DEBUG } 17:23:04 DEBUG } 17:23:04 DEBUG } 17:23:04 DEBUG } 17:23:04 INFO 17:23:04 INFO [loop_until]: kubectl --namespace=xlou top node 17:23:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543784" 17:23:04 INFO 17:23:04 INFO [loop_until]: kubectl --namespace=xlou top pods 17:23:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543784" 17:23:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543784" 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG { 17:23:04 DEBUG "status": "success", 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 DEBUG "data": { 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 DEBUG --- status code --- 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 DEBUG "resultType": "vector", 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 DEBUG --- status code --- 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG --- status code --- 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 INFO [http_cmd]: http status code OK 17:23:04 DEBUG "result": [ 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG --- status code --- 17:23:04 DEBUG { 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:23:04 DEBUG "metric": { 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG --- http response --- 17:23:04 DEBUG "pod": "ds-cts-0" 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG { 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:23:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:23:04 DEBUG }, 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG { 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG { 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG { 17:23:04 DEBUG { 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG ----- output ----- 17:23:04 DEBUG "value": [ 17:23:04 DEBUG { 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG { 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "data": { 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG { 17:23:04 DEBUG { 17:23:04 DEBUG { 17:23:04 DEBUG { 17:23:04 DEBUG 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "data": { 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "data": { 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "data": { 17:23:04 DEBUG "data": { 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "status": "success", 17:23:04 DEBUG "data": { 17:23:04 DEBUG "data": { 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "data": { 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "result": [ 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "data": { 17:23:04 DEBUG "data": { 17:23:04 DEBUG "data": { 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "result": [ 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "result": [ 17:23:04 DEBUG { 17:23:04 DEBUG "result": [ 17:23:04 DEBUG "result": [ 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "resultType": "vector", 17:23:04 DEBUG "result": [ 17:23:04 DEBUG 17:23:04 DEBUG "result": [ 17:23:04 DEBUG { 17:23:04 DEBUG "result": [ 17:23:04 DEBUG { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG { 17:23:04 DEBUG { 17:23:04 DEBUG "result": [ 17:23:04 DEBUG "result": [ 17:23:04 DEBUG "result": [ 17:23:04 DEBUG { 17:23:04 DEBUG [...] 17:23:04 DEBUG { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG { 17:23:04 DEBUG { 17:23:04 DEBUG { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:23:04 DEBUG }, 17:23:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:23:04 DEBUG }, 17:23:04 DEBUG "pod": "ds-idrepo-1" 17:23:04 DEBUG "pod": "ds-idrepo-0" 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "instance": "10.142.0.107:9100" 17:23:04 DEBUG 17:23:04 DEBUG "instance": "10.142.0.107:9100" 17:23:04 DEBUG "metric": { 17:23:04 DEBUG "value": [ 17:23:04 DEBUG }, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG }, 17:23:04 DEBUG }, 17:23:04 DEBUG "pod": "ds-idrepo-1" 17:23:04 DEBUG "pod": "ds-idrepo-1" 17:23:04 DEBUG "instance": "10.142.0.107:9100" 17:23:04 DEBUG }, 17:23:04 DEBUG 17:23:04 DEBUG }, 17:23:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:23:04 DEBUG 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 17:23:04 DEBUG "value": [ 17:23:04 DEBUG "value": [ 17:23:04 DEBUG }, 17:23:04 DEBUG }, 17:23:04 DEBUG }, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG "pod": "ds-cts-2" 17:23:04 DEBUG "value": [ 17:23:04 DEBUG }, 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG "value": [ 17:23:04 DEBUG "value": [ 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 17:23:04 DEBUG }, 17:23:04 DEBUG 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG "3.7666666666666666" 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG ] 17:23:04 DEBUG "pod": "ds-idrepo-2" 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:23:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:23:04 DEBUG 17:23:04 DEBUG [...] 17:23:04 DEBUG 17:23:04 DEBUG } 17:23:04 DEBUG }, 17:23:04 DEBUG 17:23:04 DEBUG "instance": "10.142.0.43:9100" 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:23:04 DEBUG }, 17:23:04 DEBUG }, 17:23:04 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG ] 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 17:23:04 DEBUG }, 17:23:04 DEBUG "instance": "10.142.0.43:9100" 17:23:04 DEBUG 17:23:04 DEBUG }, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG "value": [ 17:23:04 DEBUG }, 17:23:04 DEBUG 17:23:04 DEBUG 17:23:04 DEBUG } 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG "instance": "10.142.0.43:9100" 17:23:04 DEBUG "value": [ 17:23:04 DEBUG }, 17:23:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG 17:23:04 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:23:04 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 17:23:04 DEBUG "0" 17:23:04 DEBUG }, 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG "value": [ 17:23:04 DEBUG }, 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG "7.88236821775502" 17:23:04 DEBUG "2332.9666666666667" 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG }, 17:23:04 DEBUG } 17:23:04 DEBUG }, 17:23:04 DEBUG ] 17:23:04 DEBUG "value": [ 17:23:04 DEBUG "0.06510000000000389" 17:23:04 DEBUG 1713543784, 17:23:04 DEBUG "value": [ 17:23:05 DEBUG 1713543784, 17:23:04 DEBUG ] 17:23:04 DEBUG ] 17:23:04 DEBUG "0" 17:23:04 DEBUG "value": [ 17:23:04 DEBUG "value": [ 17:23:04 DEBUG } 17:23:04 DEBUG 1713543784, 17:23:05 DEBUG ] 17:23:05 DEBUG "56561494.56666666" 17:23:05 DEBUG "0.023" 17:23:04 DEBUG "2333.3333333333335" 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG ] 17:23:05 DEBUG 1713543784, 17:23:05 DEBUG 1713543784, 17:23:05 DEBUG ] 17:23:05 DEBUG "56554193.1" 17:23:05 DEBUG } 17:23:05 DEBUG ] 17:23:05 DEBUG ] 17:23:05 DEBUG ] 17:23:05 DEBUG ] 17:23:05 DEBUG ] 17:23:05 DEBUG } 17:23:05 DEBUG "0" 17:23:05 DEBUG "0" 17:23:05 DEBUG } 17:23:05 DEBUG ] 17:23:05 DEBUG ] 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG ] 17:23:05 DEBUG ] 17:23:05 DEBUG ] 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG ] 17:23:05 DEBUG ] 17:23:05 DEBUG ] 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG ] 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG ] 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG ] 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 DEBUG } 17:23:05 INFO [loop_until]: OK (rc = 0) 17:23:05 DEBUG --- stdout --- 17:23:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6173Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5371m 17% 6991Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 49m 0% 2689Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11446m 38% 11996Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9494m 31% 15449Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5226Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11536m 38% 11647Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12239m 40% 2964Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 12970m 43% 11000Mi 9% gke-xlou-cdm-ds-0f601751-j488 6247m 20% 6658Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9544m 31% 17190Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6219m 20% 7189Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9723m 32% 14888Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8030m 50% 4636Mi 7% 17:23:05 DEBUG --- stderr --- 17:23:05 DEBUG 17:23:05 INFO [loop_until]: OK (rc = 0) 17:23:05 DEBUG --- stdout --- 17:23:05 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11505m 8726Mi am-6fd554f8f7-btd4s 11448m 8773Mi am-6fd554f8f7-wxcsj 11804m 8781Mi ds-cts-0 9291m 12438Mi ds-cts-1 9568m 12377Mi ds-cts-2 9843m 12422Mi ds-idrepo-0 5588m 3420Mi ds-idrepo-1 5400m 3372Mi ds-idrepo-2 5518m 3424Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7993m 854Mi 17:23:05 DEBUG --- stderr --- 17:23:05 DEBUG 17:24:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543843" 17:24:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543843" 17:24:03 ERROR [http_cmd]: ERROR 17:24:03 INFO [http_cmd]: http status code OK 17:24:03 ERROR --- status code --- 17:24:03 DEBUG --- status code --- 17:24:03 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:24:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:03 ERROR --- http response --- 17:24:03 DEBUG --- http response --- 17:24:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:24:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:24:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:24:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543843" 17:24:03 INFO [http_cmd]: http status code OK 17:24:03 DEBUG --- status code --- 17:24:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:03 DEBUG --- http response --- 17:24:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:24:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543843" 17:24:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543843" 17:24:03 INFO [http_cmd]: http status code OK 17:24:03 DEBUG --- status code --- 17:24:03 INFO [http_cmd]: http status code OK 17:24:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:03 DEBUG --- status code --- 17:24:03 DEBUG --- http response --- 17:24:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:24:03 DEBUG --- http response --- 17:24:03 DEBUG ----- output ----- 17:24:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:24:03 DEBUG { 17:24:03 DEBUG ----- output ----- 17:24:03 DEBUG "status": "success", 17:24:03 DEBUG { 17:24:03 DEBUG "data": { 17:24:03 DEBUG "status": "success", 17:24:03 DEBUG "resultType": "vector", 17:24:03 DEBUG "data": { 17:24:03 DEBUG "result": [ 17:24:03 DEBUG "resultType": "vector", 17:24:03 DEBUG { 17:24:03 DEBUG "result": [ 17:24:03 DEBUG "metric": { 17:24:03 DEBUG { 17:24:03 DEBUG "pod": "ds-idrepo-0" 17:24:03 DEBUG "metric": { 17:24:03 DEBUG }, 17:24:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:24:03 DEBUG "value": [ 17:24:03 DEBUG }, 17:24:03 DEBUG 17:24:03 DEBUG "value": [ 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG [...] 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG [...] 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG "pod": "ds-cts-2" 17:24:03 DEBUG 17:24:03 DEBUG }, 17:24:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:24:03 DEBUG "value": [ 17:24:03 DEBUG }, 17:24:03 DEBUG 1713543843, 17:24:03 DEBUG "value": [ 17:24:03 DEBUG "0" 17:24:03 DEBUG 1713543843, 17:24:03 DEBUG ] 17:24:03 DEBUG "2331.5666666666666" 17:24:03 DEBUG } 17:24:03 DEBUG ] 17:24:03 DEBUG ] 17:24:03 DEBUG } 17:24:03 DEBUG } 17:24:03 DEBUG ] 17:24:03 DEBUG } 17:24:03 DEBUG } 17:24:03 DEBUG } 17:24:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543843" 17:24:03 INFO [http_cmd]: http status code OK 17:24:03 DEBUG --- status code --- 17:24:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:03 DEBUG --- http response --- 17:24:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:24:03 DEBUG ----- output ----- 17:24:03 DEBUG { 17:24:03 DEBUG "status": "success", 17:24:03 DEBUG "data": { 17:24:03 DEBUG "resultType": "vector", 17:24:03 DEBUG "result": [ 17:24:03 DEBUG { 17:24:03 DEBUG "metric": { 17:24:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:24:03 DEBUG }, 17:24:03 DEBUG "value": [ 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG [...] 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:24:03 DEBUG }, 17:24:03 DEBUG "value": [ 17:24:03 DEBUG 1713543843, 17:24:03 DEBUG "0" 17:24:03 DEBUG ] 17:24:03 DEBUG } 17:24:03 DEBUG ] 17:24:03 DEBUG } 17:24:03 DEBUG } 17:24:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543843" 17:24:03 INFO [http_cmd]: http status code OK 17:24:03 DEBUG --- status code --- 17:24:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:03 DEBUG --- http response --- 17:24:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:24:03 DEBUG ----- output ----- 17:24:03 DEBUG { 17:24:03 DEBUG "status": "success", 17:24:03 DEBUG "data": { 17:24:03 DEBUG "resultType": "vector", 17:24:03 DEBUG "result": [ 17:24:03 DEBUG { 17:24:03 DEBUG "metric": { 17:24:03 DEBUG "pod": "ds-idrepo-0" 17:24:03 DEBUG }, 17:24:03 DEBUG "value": [ 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG [...] 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:24:03 DEBUG }, 17:24:03 DEBUG "value": [ 17:24:03 DEBUG 1713543843, 17:24:03 DEBUG "0" 17:24:03 DEBUG ] 17:24:03 DEBUG } 17:24:03 DEBUG ] 17:24:03 DEBUG } 17:24:03 DEBUG } 17:24:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543843" 17:24:03 INFO [http_cmd]: http status code OK 17:24:03 DEBUG --- status code --- 17:24:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:03 DEBUG --- http response --- 17:24:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:24:03 DEBUG ----- output ----- 17:24:03 DEBUG { 17:24:03 DEBUG "status": "success", 17:24:03 DEBUG "data": { 17:24:03 DEBUG "resultType": "vector", 17:24:03 DEBUG "result": [ 17:24:03 DEBUG { 17:24:03 DEBUG "metric": { 17:24:03 DEBUG "instance": "10.142.0.107:9100" 17:24:03 DEBUG }, 17:24:03 DEBUG "value": [ 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG [...] 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG "instance": "10.142.0.43:9100" 17:24:03 DEBUG }, 17:24:03 DEBUG "value": [ 17:24:03 DEBUG 1713543843, 17:24:03 DEBUG "0.04766666666666311" 17:24:03 DEBUG ] 17:24:03 DEBUG } 17:24:03 DEBUG ] 17:24:03 DEBUG } 17:24:03 DEBUG } 17:24:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543843" 17:24:03 INFO [http_cmd]: http status code OK 17:24:03 DEBUG --- status code --- 17:24:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:03 DEBUG --- http response --- 17:24:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:24:03 DEBUG ----- output ----- 17:24:03 DEBUG { 17:24:03 DEBUG "status": "success", 17:24:03 DEBUG "data": { 17:24:03 DEBUG "resultType": "vector", 17:24:03 DEBUG "result": [ 17:24:03 DEBUG { 17:24:03 DEBUG "metric": { 17:24:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:24:03 DEBUG }, 17:24:03 DEBUG "value": [ 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG [...] 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG 17:24:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:24:03 DEBUG }, 17:24:03 DEBUG "value": [ 17:24:03 DEBUG 1713543843, 17:24:03 DEBUG "3325779" 17:24:03 DEBUG ] 17:24:03 DEBUG } 17:24:03 DEBUG ] 17:24:03 DEBUG } 17:24:03 DEBUG } 17:24:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543844" 17:24:04 INFO [http_cmd]: http status code OK 17:24:04 DEBUG --- status code --- 17:24:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:04 DEBUG --- http response --- 17:24:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:24:04 DEBUG ----- output ----- 17:24:04 DEBUG { 17:24:04 DEBUG "status": "success", 17:24:04 DEBUG "data": { 17:24:04 DEBUG "resultType": "vector", 17:24:04 DEBUG "result": [ 17:24:04 DEBUG { 17:24:04 DEBUG "metric": { 17:24:04 DEBUG "instance": "10.142.0.107:9100" 17:24:04 DEBUG }, 17:24:04 DEBUG "value": [ 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG [...] 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG "instance": "10.142.0.43:9100" 17:24:04 DEBUG }, 17:24:04 DEBUG "value": [ 17:24:04 DEBUG 1713543844, 17:24:04 DEBUG "1691538334720" 17:24:04 DEBUG ] 17:24:04 DEBUG } 17:24:04 DEBUG ] 17:24:04 DEBUG } 17:24:04 DEBUG } 17:24:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543844" 17:24:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543844" 17:24:04 INFO [http_cmd]: http status code OK 17:24:04 DEBUG --- status code --- 17:24:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:04 DEBUG --- http response --- 17:24:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:24:04 DEBUG ----- output ----- 17:24:04 DEBUG { 17:24:04 DEBUG "status": "success", 17:24:04 DEBUG "data": { 17:24:04 DEBUG "resultType": "vector", 17:24:04 DEBUG "result": [ 17:24:04 DEBUG { 17:24:04 DEBUG "metric": { 17:24:04 DEBUG "pod": "ds-idrepo-0" 17:24:04 INFO [http_cmd]: http status code OK 17:24:04 DEBUG }, 17:24:04 DEBUG --- status code --- 17:24:04 DEBUG "value": [ 17:24:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:04 DEBUG 17:24:04 DEBUG --- http response --- 17:24:04 DEBUG 17:24:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:24:04 DEBUG 17:24:04 DEBUG ----- output ----- 17:24:04 DEBUG [...] 17:24:04 DEBUG { 17:24:04 DEBUG 17:24:04 DEBUG "status": "success", 17:24:04 DEBUG 17:24:04 DEBUG "data": { 17:24:04 DEBUG 17:24:04 DEBUG "resultType": "vector", 17:24:04 DEBUG "pod": "ds-idrepo-2" 17:24:04 DEBUG "result": [ 17:24:04 DEBUG }, 17:24:04 DEBUG { 17:24:04 DEBUG "value": [ 17:24:04 DEBUG "metric": { 17:24:04 DEBUG 1713543844, 17:24:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:24:04 DEBUG "0" 17:24:04 DEBUG }, 17:24:04 DEBUG ] 17:24:04 DEBUG "value": [ 17:24:04 DEBUG } 17:24:04 DEBUG 17:24:04 DEBUG ] 17:24:04 DEBUG 17:24:04 DEBUG } 17:24:04 DEBUG 17:24:04 DEBUG } 17:24:04 DEBUG [...] 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:24:04 DEBUG }, 17:24:04 DEBUG "value": [ 17:24:04 DEBUG 1713543844, 17:24:04 DEBUG "994611200" 17:24:04 DEBUG ] 17:24:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543844" 17:24:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543844" 17:24:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543844" 17:24:04 DEBUG } 17:24:04 DEBUG ] 17:24:04 DEBUG } 17:24:04 DEBUG } 17:24:04 INFO [http_cmd]: http status code OK 17:24:04 DEBUG --- status code --- 17:24:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:04 DEBUG --- http response --- 17:24:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:24:04 DEBUG ----- output ----- 17:24:04 DEBUG { 17:24:04 DEBUG "status": "success", 17:24:04 DEBUG "data": { 17:24:04 DEBUG "resultType": "vector", 17:24:04 DEBUG "result": [ 17:24:04 DEBUG { 17:24:04 DEBUG "metric": { 17:24:04 DEBUG "pod": "ds-idrepo-0" 17:24:04 DEBUG }, 17:24:04 DEBUG "value": [ 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG [...] 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG "pod": "ds-idrepo-2" 17:24:04 DEBUG }, 17:24:04 DEBUG "value": [ 17:24:04 DEBUG 1713543844, 17:24:04 DEBUG "0" 17:24:04 DEBUG ] 17:24:04 DEBUG } 17:24:04 DEBUG ] 17:24:04 DEBUG } 17:24:04 DEBUG } 17:24:04 INFO [http_cmd]: http status code OK 17:24:04 INFO [http_cmd]: http status code OK 17:24:04 DEBUG --- status code --- 17:24:04 DEBUG --- status code --- 17:24:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:04 DEBUG --- http response --- 17:24:04 DEBUG --- http response --- 17:24:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:24:04 DEBUG ----- output ----- 17:24:04 DEBUG { 17:24:04 DEBUG "status": "success", 17:24:04 DEBUG "data": { 17:24:04 DEBUG "resultType": "vector", 17:24:04 DEBUG "result": [ 17:24:04 DEBUG { 17:24:04 DEBUG "metric": { 17:24:04 DEBUG "pod": "ds-cts-0" 17:24:04 DEBUG }, 17:24:04 DEBUG "value": [ 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:24:04 DEBUG [...] 17:24:04 DEBUG ----- output ----- 17:24:04 DEBUG 17:24:04 DEBUG { 17:24:04 DEBUG 17:24:04 DEBUG "status": "success", 17:24:04 DEBUG 17:24:04 DEBUG "data": { 17:24:04 DEBUG "pod": "ds-cts-2" 17:24:04 DEBUG "resultType": "vector", 17:24:04 DEBUG }, 17:24:04 DEBUG "result": [ 17:24:04 DEBUG "value": [ 17:24:04 DEBUG { 17:24:04 DEBUG 1713543844, 17:24:04 DEBUG "metric": { 17:24:04 DEBUG "0.0010077133333333333" 17:24:04 DEBUG "pod": "ds-cts-0" 17:24:04 DEBUG ] 17:24:04 DEBUG }, 17:24:04 DEBUG } 17:24:04 DEBUG "value": [ 17:24:04 DEBUG ] 17:24:04 DEBUG 17:24:04 DEBUG } 17:24:04 DEBUG 17:24:04 DEBUG } 17:24:04 DEBUG 17:24:04 DEBUG [...] 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG 17:24:04 DEBUG "pod": "ds-cts-2" 17:24:04 DEBUG }, 17:24:04 DEBUG "value": [ 17:24:04 DEBUG 1713543844, 17:24:04 DEBUG "0" 17:24:04 DEBUG ] 17:24:04 DEBUG } 17:24:04 DEBUG ] 17:24:04 DEBUG } 17:24:04 DEBUG } 17:24:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543845" 17:24:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543845" 17:24:05 INFO [http_cmd]: http status code OK 17:24:05 DEBUG --- status code --- 17:24:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:05 DEBUG --- http response --- 17:24:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:24:05 DEBUG ----- output ----- 17:24:05 DEBUG { 17:24:05 DEBUG "status": "success", 17:24:05 DEBUG "data": { 17:24:05 DEBUG "resultType": "vector", 17:24:05 DEBUG "result": [ 17:24:05 DEBUG { 17:24:05 DEBUG "metric": { 17:24:05 DEBUG "instance": "10.142.0.107:9100" 17:24:05 DEBUG }, 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG [...] 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG "instance": "10.142.0.43:9100" 17:24:05 DEBUG }, 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 1713543845, 17:24:05 DEBUG "56500988.333333336" 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG } 17:24:05 INFO [http_cmd]: http status code OK 17:24:05 DEBUG --- status code --- 17:24:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:05 DEBUG --- http response --- 17:24:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:24:05 DEBUG ----- output ----- 17:24:05 DEBUG { 17:24:05 DEBUG "status": "success", 17:24:05 DEBUG "data": { 17:24:05 DEBUG "resultType": "vector", 17:24:05 DEBUG "result": [ 17:24:05 DEBUG { 17:24:05 DEBUG "metric": { 17:24:05 DEBUG "pod": "ds-idrepo-0" 17:24:05 DEBUG }, 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG [...] 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:24:05 DEBUG }, 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 1713543845, 17:24:05 DEBUG "0" 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG } 17:24:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543845" 17:24:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543845" 17:24:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543845" 17:24:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543845" 17:24:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543845" 17:24:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543845" 17:24:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543845" 17:24:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543845" 17:24:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543845" 17:24:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543845" 17:24:05 INFO [http_cmd]: http status code OK 17:24:05 INFO 17:24:05 DEBUG --- status code --- 17:24:05 INFO [loop_until]: kubectl --namespace=xlou top node 17:24:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:24:05 INFO [http_cmd]: http status code OK 17:24:05 DEBUG --- http response --- 17:24:05 INFO [http_cmd]: http status code OK 17:24:05 DEBUG --- status code --- 17:24:05 INFO [http_cmd]: http status code OK 17:24:05 INFO [http_cmd]: http status code OK 17:24:05 DEBUG --- status code --- 17:24:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:24:05 INFO [http_cmd]: http status code OK 17:24:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:05 INFO [http_cmd]: http status code OK 17:24:05 DEBUG --- status code --- 17:24:05 INFO [http_cmd]: http status code OK 17:24:05 INFO [http_cmd]: http status code OK 17:24:05 DEBUG --- status code --- 17:24:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:05 DEBUG ----- output ----- 17:24:05 DEBUG --- status code --- 17:24:05 DEBUG --- http response --- 17:24:05 DEBUG --- status code --- 17:24:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:05 DEBUG --- status code --- 17:24:05 DEBUG --- status code --- 17:24:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:05 DEBUG --- http response --- 17:24:05 DEBUG { 17:24:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:05 INFO [http_cmd]: http status code OK 17:24:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:24:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:05 DEBUG --- http response --- 17:24:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:05 DEBUG --- http response --- 17:24:05 DEBUG --- http response --- 17:24:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:24:05 DEBUG "status": "success", 17:24:05 DEBUG --- http response --- 17:24:05 DEBUG --- status code --- 17:24:05 DEBUG ----- output ----- 17:24:05 DEBUG --- http response --- 17:24:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:24:05 INFO 17:24:05 DEBUG ----- output ----- 17:24:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:24:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:24:05 DEBUG "data": { 17:24:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:24:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:24:05 DEBUG { 17:24:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:24:05 DEBUG --- http response --- 17:24:05 DEBUG ----- output ----- 17:24:05 INFO [loop_until]: kubectl --namespace=xlou top pods 17:24:05 DEBUG { 17:24:05 DEBUG ----- output ----- 17:24:05 DEBUG ----- output ----- 17:24:05 DEBUG { 17:24:05 DEBUG ----- output ----- 17:24:05 DEBUG --- http response --- 17:24:05 DEBUG "status": "success", 17:24:05 DEBUG ----- output ----- 17:24:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:24:05 DEBUG { 17:24:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:24:05 DEBUG "status": "success", 17:24:05 DEBUG { 17:24:05 DEBUG "resultType": "vector", 17:24:05 DEBUG "result": [ 17:24:05 DEBUG { 17:24:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:24:05 DEBUG "data": { 17:24:05 DEBUG { 17:24:05 DEBUG ----- output ----- 17:24:05 DEBUG "status": "success", 17:24:05 DEBUG "data": { 17:24:05 DEBUG "status": "success", 17:24:05 DEBUG "status": "success", 17:24:05 DEBUG "data": { 17:24:05 DEBUG "status": "success", 17:24:05 DEBUG ----- output ----- 17:24:05 DEBUG "resultType": "vector", 17:24:05 DEBUG "status": "success", 17:24:05 DEBUG "data": { 17:24:05 DEBUG "data": { 17:24:05 DEBUG "resultType": "vector", 17:24:05 DEBUG "data": { 17:24:05 DEBUG "resultType": "vector", 17:24:05 DEBUG { 17:24:05 DEBUG "data": { 17:24:05 DEBUG { 17:24:05 DEBUG "result": [ 17:24:05 DEBUG { 17:24:05 DEBUG "metric": { 17:24:05 DEBUG "instance": "10.142.0.107:9100" 17:24:05 DEBUG "result": [ 17:24:05 DEBUG "resultType": "vector", 17:24:05 DEBUG "result": [ 17:24:05 DEBUG { 17:24:05 DEBUG "metric": { 17:24:05 DEBUG "status": "success", 17:24:05 DEBUG "resultType": "vector", 17:24:05 DEBUG }, 17:24:05 DEBUG { 17:24:05 DEBUG "resultType": "vector", 17:24:05 DEBUG { 17:24:05 DEBUG "resultType": "vector", 17:24:05 DEBUG "metric": { 17:24:05 DEBUG "result": [ 17:24:05 DEBUG { 17:24:05 DEBUG "data": { 17:24:05 DEBUG "result": [ 17:24:05 DEBUG "value": [ 17:24:05 DEBUG "status": "success", 17:24:05 DEBUG "result": [ 17:24:05 DEBUG "metric": { 17:24:05 DEBUG "result": [ 17:24:05 DEBUG "pod": "ds-idrepo-0" 17:24:05 DEBUG "metric": { 17:24:05 DEBUG "pod": "ds-idrepo-0" 17:24:05 DEBUG "resultType": "vector", 17:24:05 DEBUG { 17:24:05 DEBUG 17:24:05 DEBUG "data": { 17:24:05 DEBUG { 17:24:05 DEBUG "metric": { 17:24:05 DEBUG { 17:24:05 DEBUG "metric": { 17:24:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:24:05 DEBUG }, 17:24:05 DEBUG "result": [ 17:24:05 DEBUG "metric": { 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG "pod": "ds-cts-0" 17:24:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:24:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:24:05 DEBUG }, 17:24:05 DEBUG "value": [ 17:24:05 DEBUG "value": [ 17:24:05 DEBUG { 17:24:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:24:05 DEBUG [...] 17:24:05 DEBUG "resultType": "vector", 17:24:05 DEBUG }, 17:24:05 DEBUG }, 17:24:05 DEBUG 17:24:05 DEBUG }, 17:24:05 DEBUG }, 17:24:05 DEBUG 17:24:05 DEBUG "metric": { 17:24:05 DEBUG }, 17:24:05 DEBUG 17:24:05 DEBUG "result": [ 17:24:05 DEBUG "value": [ 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 17:24:05 DEBUG "value": [ 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 17:24:05 DEBUG "instance": "10.142.0.107:9100" 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 17:24:05 DEBUG { 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG }, 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG "instance": "10.142.0.43:9100" 17:24:05 DEBUG }, 17:24:05 DEBUG 17:24:05 DEBUG [...] 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG [...] 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG "metric": { 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG [...] 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 17:24:05 DEBUG "pod": "ds-idrepo-0" 17:24:05 DEBUG [...] 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG [...] 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 1713543845, 17:24:05 DEBUG [...] 17:24:05 DEBUG }, 17:24:05 DEBUG 17:24:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:24:05 DEBUG [...] 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG "0.06756666666662738" 17:24:05 DEBUG 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 17:24:05 DEBUG }, 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG "pod": "ds-idrepo-2" 17:24:05 DEBUG [...] 17:24:05 DEBUG 17:24:05 DEBUG ] 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG }, 17:24:05 DEBUG 17:24:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:24:05 DEBUG } 17:24:05 DEBUG "pod": "ds-cts-2" 17:24:05 DEBUG }, 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 1713543845, 17:24:05 DEBUG 17:24:05 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:24:05 DEBUG }, 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 1713543845, 17:24:05 DEBUG ] 17:24:05 DEBUG 17:24:05 DEBUG "3.2" 17:24:05 DEBUG 1713543845, 17:24:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:24:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:24:05 DEBUG }, 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 17:24:05 DEBUG "value": [ 17:24:05 DEBUG } 17:24:05 DEBUG 17:24:05 DEBUG ] 17:24:05 DEBUG "7.852000488144725" 17:24:05 DEBUG }, 17:24:05 DEBUG }, 17:24:05 DEBUG 1713543845, 17:24:05 DEBUG "2331.866666666667" 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG } 17:24:05 DEBUG 17:24:05 DEBUG } 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG "value": [ 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG 17:24:05 DEBUG 1713543845, 17:24:05 DEBUG [...] 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 17:24:05 DEBUG 1713543845, 17:24:05 DEBUG "0.023" 17:24:05 DEBUG ] 17:24:05 DEBUG "instance": "10.142.0.43:9100" 17:24:05 DEBUG }, 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 1713543845, 17:24:05 DEBUG "56494093.96666667" 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG "0.00001473228181888893" 17:24:05 DEBUG } 17:24:05 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:24:05 DEBUG } 17:24:05 DEBUG ] 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG } 17:24:05 DEBUG "0" 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG } 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG } 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG } 17:24:05 DEBUG } 17:24:05 DEBUG } 17:24:05 DEBUG }, 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 1713543845, 17:24:05 DEBUG "0" 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG } 17:24:05 DEBUG } 17:24:05 DEBUG "value": [ 17:24:05 DEBUG 1713543845, 17:24:05 DEBUG "2331.7666666666664" 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG ] 17:24:05 DEBUG } 17:24:05 DEBUG } 17:24:05 INFO [loop_until]: OK (rc = 0) 17:24:05 DEBUG --- stdout --- 17:24:05 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11702m 8729Mi am-6fd554f8f7-btd4s 11402m 8775Mi am-6fd554f8f7-wxcsj 12201m 8782Mi ds-cts-0 9125m 12438Mi ds-cts-1 9629m 12377Mi ds-cts-2 9703m 12422Mi ds-idrepo-0 5609m 3420Mi ds-idrepo-1 5454m 3372Mi ds-idrepo-2 5414m 3424Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7811m 856Mi 17:24:05 DEBUG --- stderr --- 17:24:05 DEBUG 17:24:05 INFO [loop_until]: OK (rc = 0) 17:24:05 DEBUG --- stdout --- 17:24:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6183Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5407m 18% 6988Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 48m 0% 2691Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11232m 37% 12007Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9400m 31% 15457Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 136m 0% 5240Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11513m 38% 11654Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12265m 41% 2965Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13025m 43% 10985Mi 9% gke-xlou-cdm-ds-0f601751-j488 6181m 20% 6658Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9737m 32% 17181Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6169m 20% 7195Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9731m 32% 14888Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7982m 50% 4635Mi 7% 17:24:05 DEBUG --- stderr --- 17:24:05 DEBUG 17:25:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543903" 17:25:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543903" 17:25:03 ERROR [http_cmd]: ERROR 17:25:03 INFO [http_cmd]: http status code OK 17:25:03 ERROR --- status code --- 17:25:03 DEBUG --- status code --- 17:25:03 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:25:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:03 ERROR --- http response --- 17:25:03 DEBUG --- http response --- 17:25:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:25:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:25:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:25:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543903" 17:25:03 INFO [http_cmd]: http status code OK 17:25:03 DEBUG --- status code --- 17:25:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:03 DEBUG --- http response --- 17:25:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:25:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543903" 17:25:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543903" 17:25:03 INFO [http_cmd]: http status code OK 17:25:03 INFO [http_cmd]: http status code OK 17:25:03 DEBUG --- status code --- 17:25:03 DEBUG --- status code --- 17:25:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:03 DEBUG --- http response --- 17:25:03 DEBUG --- http response --- 17:25:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:25:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:25:03 DEBUG ----- output ----- 17:25:03 DEBUG ----- output ----- 17:25:03 DEBUG { 17:25:03 DEBUG { 17:25:03 DEBUG "status": "success", 17:25:03 DEBUG "status": "success", 17:25:03 DEBUG "data": { 17:25:03 DEBUG "data": { 17:25:03 DEBUG "resultType": "vector", 17:25:03 DEBUG "resultType": "vector", 17:25:03 DEBUG "result": [ 17:25:03 DEBUG "result": [ 17:25:03 DEBUG { 17:25:03 DEBUG { 17:25:03 DEBUG "metric": { 17:25:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:25:03 DEBUG "metric": { 17:25:03 DEBUG }, 17:25:03 DEBUG "pod": "ds-idrepo-0" 17:25:03 DEBUG "value": [ 17:25:03 DEBUG }, 17:25:03 DEBUG 17:25:03 DEBUG "value": [ 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG [...] 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG [...] 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:25:03 DEBUG 17:25:03 DEBUG }, 17:25:03 DEBUG "pod": "ds-cts-2" 17:25:03 DEBUG "value": [ 17:25:03 DEBUG }, 17:25:03 DEBUG 1713543903, 17:25:03 DEBUG "value": [ 17:25:03 DEBUG "2334.6666666666665" 17:25:03 DEBUG 1713543903, 17:25:03 DEBUG ] 17:25:03 DEBUG "0" 17:25:03 DEBUG } 17:25:03 DEBUG ] 17:25:03 DEBUG ] 17:25:03 DEBUG } 17:25:03 DEBUG } 17:25:03 DEBUG ] 17:25:03 DEBUG } 17:25:03 DEBUG } 17:25:03 DEBUG } 17:25:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543903" 17:25:03 INFO [http_cmd]: http status code OK 17:25:03 DEBUG --- status code --- 17:25:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:03 DEBUG --- http response --- 17:25:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:25:03 DEBUG ----- output ----- 17:25:03 DEBUG { 17:25:03 DEBUG "status": "success", 17:25:03 DEBUG "data": { 17:25:03 DEBUG "resultType": "vector", 17:25:03 DEBUG "result": [ 17:25:03 DEBUG { 17:25:03 DEBUG "metric": { 17:25:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:25:03 DEBUG }, 17:25:03 DEBUG "value": [ 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG [...] 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:25:03 DEBUG }, 17:25:03 DEBUG "value": [ 17:25:03 DEBUG 1713543903, 17:25:03 DEBUG "0" 17:25:03 DEBUG ] 17:25:03 DEBUG } 17:25:03 DEBUG ] 17:25:03 DEBUG } 17:25:03 DEBUG } 17:25:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543903" 17:25:03 INFO [http_cmd]: http status code OK 17:25:03 DEBUG --- status code --- 17:25:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:03 DEBUG --- http response --- 17:25:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:25:03 DEBUG ----- output ----- 17:25:03 DEBUG { 17:25:03 DEBUG "status": "success", 17:25:03 DEBUG "data": { 17:25:03 DEBUG "resultType": "vector", 17:25:03 DEBUG "result": [ 17:25:03 DEBUG { 17:25:03 DEBUG "metric": { 17:25:03 DEBUG "pod": "ds-idrepo-1" 17:25:03 DEBUG }, 17:25:03 DEBUG "value": [ 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG [...] 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG "pod": "idm-b85f9d978-txpwv" 17:25:03 DEBUG }, 17:25:03 DEBUG "value": [ 17:25:03 DEBUG 1713543903, 17:25:03 DEBUG "0" 17:25:03 DEBUG ] 17:25:03 DEBUG } 17:25:03 DEBUG ] 17:25:03 DEBUG } 17:25:03 DEBUG } 17:25:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543903" 17:25:03 INFO [http_cmd]: http status code OK 17:25:03 DEBUG --- status code --- 17:25:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:03 DEBUG --- http response --- 17:25:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:25:03 DEBUG ----- output ----- 17:25:03 DEBUG { 17:25:03 DEBUG "status": "success", 17:25:03 DEBUG "data": { 17:25:03 DEBUG "resultType": "vector", 17:25:03 DEBUG "result": [ 17:25:03 DEBUG { 17:25:03 DEBUG "metric": { 17:25:03 DEBUG "instance": "10.142.0.107:9100" 17:25:03 DEBUG }, 17:25:03 DEBUG "value": [ 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG [...] 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG 17:25:03 DEBUG "instance": "10.142.0.43:9100" 17:25:03 DEBUG }, 17:25:03 DEBUG "value": [ 17:25:03 DEBUG 1713543903, 17:25:03 DEBUG "0.04633333333333856" 17:25:03 DEBUG ] 17:25:03 DEBUG } 17:25:03 DEBUG ] 17:25:03 DEBUG } 17:25:03 DEBUG } 17:25:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543904" 17:25:04 INFO [http_cmd]: http status code OK 17:25:04 DEBUG --- status code --- 17:25:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:04 DEBUG --- http response --- 17:25:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:25:04 DEBUG ----- output ----- 17:25:04 DEBUG { 17:25:04 DEBUG "status": "success", 17:25:04 DEBUG "data": { 17:25:04 DEBUG "resultType": "vector", 17:25:04 DEBUG "result": [ 17:25:04 DEBUG { 17:25:04 DEBUG "metric": { 17:25:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:25:04 DEBUG }, 17:25:04 DEBUG "value": [ 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG [...] 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:25:04 DEBUG }, 17:25:04 DEBUG "value": [ 17:25:04 DEBUG 1713543904, 17:25:04 DEBUG "3465482.5" 17:25:04 DEBUG ] 17:25:04 DEBUG } 17:25:04 DEBUG ] 17:25:04 DEBUG } 17:25:04 DEBUG } 17:25:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543904" 17:25:04 INFO [http_cmd]: http status code OK 17:25:04 DEBUG --- status code --- 17:25:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:04 DEBUG --- http response --- 17:25:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:25:04 DEBUG ----- output ----- 17:25:04 DEBUG { 17:25:04 DEBUG "status": "success", 17:25:04 DEBUG "data": { 17:25:04 DEBUG "resultType": "vector", 17:25:04 DEBUG "result": [ 17:25:04 DEBUG { 17:25:04 DEBUG "metric": { 17:25:04 DEBUG "instance": "10.142.0.107:9100" 17:25:04 DEBUG }, 17:25:04 DEBUG "value": [ 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG [...] 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG "instance": "10.142.0.43:9100" 17:25:04 DEBUG }, 17:25:04 DEBUG "value": [ 17:25:04 DEBUG 1713543904, 17:25:04 DEBUG "1694371646464" 17:25:04 DEBUG ] 17:25:04 DEBUG } 17:25:04 DEBUG ] 17:25:04 DEBUG } 17:25:04 DEBUG } 17:25:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543904" 17:25:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543904" 17:25:04 INFO [http_cmd]: http status code OK 17:25:04 DEBUG --- status code --- 17:25:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:04 DEBUG --- http response --- 17:25:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:25:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543904" 17:25:04 INFO [http_cmd]: http status code OK 17:25:04 DEBUG ----- output ----- 17:25:04 DEBUG --- status code --- 17:25:04 DEBUG { 17:25:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:04 DEBUG "status": "success", 17:25:04 DEBUG --- http response --- 17:25:04 DEBUG "data": { 17:25:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:25:04 DEBUG "resultType": "vector", 17:25:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543904" 17:25:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543904" 17:25:04 DEBUG ----- output ----- 17:25:04 DEBUG "result": [ 17:25:04 DEBUG { 17:25:04 DEBUG { 17:25:04 DEBUG "status": "success", 17:25:04 DEBUG "metric": { 17:25:04 DEBUG "data": { 17:25:04 DEBUG "pod": "ds-idrepo-0" 17:25:04 DEBUG "resultType": "vector", 17:25:04 DEBUG }, 17:25:04 DEBUG "result": [ 17:25:04 DEBUG "value": [ 17:25:04 DEBUG { 17:25:04 DEBUG 17:25:04 DEBUG "metric": { 17:25:04 DEBUG 17:25:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:25:04 DEBUG 17:25:04 DEBUG }, 17:25:04 DEBUG [...] 17:25:04 DEBUG "value": [ 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG "pod": "ds-idrepo-2" 17:25:04 DEBUG [...] 17:25:04 DEBUG }, 17:25:04 DEBUG 17:25:04 DEBUG "value": [ 17:25:04 DEBUG 1713543904, 17:25:04 DEBUG "0" 17:25:04 DEBUG 17:25:04 DEBUG ] 17:25:04 DEBUG 17:25:04 DEBUG } 17:25:04 DEBUG ] 17:25:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:25:04 INFO [http_cmd]: http status code OK 17:25:04 DEBUG } 17:25:04 DEBUG }, 17:25:04 DEBUG --- status code --- 17:25:04 DEBUG } 17:25:04 DEBUG "value": [ 17:25:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:04 DEBUG 1713543904, 17:25:04 DEBUG --- http response --- 17:25:04 DEBUG "997060608" 17:25:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:25:04 DEBUG ] 17:25:04 DEBUG ----- output ----- 17:25:04 DEBUG } 17:25:04 DEBUG { 17:25:04 DEBUG ] 17:25:04 DEBUG "status": "success", 17:25:04 DEBUG "data": { 17:25:04 DEBUG "resultType": "vector", 17:25:04 DEBUG "result": [ 17:25:04 DEBUG } 17:25:04 DEBUG { 17:25:04 DEBUG } 17:25:04 DEBUG "metric": { 17:25:04 DEBUG "pod": "ds-idrepo-0" 17:25:04 DEBUG }, 17:25:04 DEBUG "value": [ 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG [...] 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG "pod": "ds-idrepo-2" 17:25:04 DEBUG }, 17:25:04 DEBUG "value": [ 17:25:04 DEBUG 1713543904, 17:25:04 DEBUG "0" 17:25:04 DEBUG ] 17:25:04 DEBUG } 17:25:04 DEBUG ] 17:25:04 DEBUG } 17:25:04 DEBUG } 17:25:04 INFO [http_cmd]: http status code OK 17:25:04 DEBUG --- status code --- 17:25:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:04 DEBUG --- http response --- 17:25:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:25:04 DEBUG ----- output ----- 17:25:04 DEBUG { 17:25:04 DEBUG "status": "success", 17:25:04 DEBUG "data": { 17:25:04 DEBUG "resultType": "vector", 17:25:04 DEBUG "result": [ 17:25:04 DEBUG { 17:25:04 DEBUG "metric": { 17:25:04 DEBUG "pod": "ds-cts-0" 17:25:04 DEBUG }, 17:25:04 DEBUG "value": [ 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 INFO [http_cmd]: http status code OK 17:25:04 DEBUG 17:25:04 DEBUG --- status code --- 17:25:04 DEBUG [...] 17:25:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:04 DEBUG 17:25:04 DEBUG --- http response --- 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG "pod": "ds-cts-2" 17:25:04 DEBUG }, 17:25:04 DEBUG "value": [ 17:25:04 DEBUG 1713543904, 17:25:04 DEBUG "0" 17:25:04 DEBUG ] 17:25:04 DEBUG } 17:25:04 DEBUG ] 17:25:04 DEBUG } 17:25:04 DEBUG } 17:25:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:25:04 DEBUG ----- output ----- 17:25:04 DEBUG { 17:25:04 DEBUG "status": "success", 17:25:04 DEBUG "data": { 17:25:04 DEBUG "resultType": "vector", 17:25:04 DEBUG "result": [ 17:25:04 DEBUG { 17:25:04 DEBUG "metric": { 17:25:04 DEBUG "pod": "ds-cts-0" 17:25:04 DEBUG }, 17:25:04 DEBUG "value": [ 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG [...] 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG 17:25:04 DEBUG "pod": "ds-cts-2" 17:25:04 DEBUG }, 17:25:04 DEBUG "value": [ 17:25:04 DEBUG 1713543904, 17:25:04 DEBUG "0.00006666666666666667" 17:25:04 DEBUG ] 17:25:04 DEBUG } 17:25:04 DEBUG ] 17:25:04 DEBUG } 17:25:04 DEBUG } 17:25:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543905" 17:25:05 INFO [http_cmd]: http status code OK 17:25:05 DEBUG --- status code --- 17:25:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:05 DEBUG --- http response --- 17:25:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:25:05 DEBUG ----- output ----- 17:25:05 DEBUG { 17:25:05 DEBUG "status": "success", 17:25:05 DEBUG "data": { 17:25:05 DEBUG "resultType": "vector", 17:25:05 DEBUG "result": [ 17:25:05 DEBUG { 17:25:05 DEBUG "metric": { 17:25:05 DEBUG "instance": "10.142.0.107:9100" 17:25:05 DEBUG }, 17:25:05 DEBUG "value": [ 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG [...] 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG "instance": "10.142.0.43:9100" 17:25:05 DEBUG }, 17:25:05 DEBUG "value": [ 17:25:05 DEBUG 1713543905, 17:25:05 DEBUG "56565415.83333333" 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543905" 17:25:05 INFO [http_cmd]: http status code OK 17:25:05 DEBUG --- status code --- 17:25:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:05 DEBUG --- http response --- 17:25:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:25:05 DEBUG ----- output ----- 17:25:05 DEBUG { 17:25:05 DEBUG "status": "success", 17:25:05 DEBUG "data": { 17:25:05 DEBUG "resultType": "vector", 17:25:05 DEBUG "result": [ 17:25:05 DEBUG { 17:25:05 DEBUG "metric": { 17:25:05 DEBUG "pod": "ds-idrepo-1" 17:25:05 DEBUG }, 17:25:05 DEBUG "value": [ 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG [...] 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:25:05 DEBUG }, 17:25:05 DEBUG "value": [ 17:25:05 DEBUG 1713543905, 17:25:05 DEBUG "3753.7394036592154" 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543905" 17:25:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543905" 17:25:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543905" 17:25:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543905" 17:25:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543905" 17:25:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543905" 17:25:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543905" 17:25:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543905" 17:25:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543905" 17:25:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543905" 17:25:05 INFO [http_cmd]: http status code OK 17:25:05 INFO [http_cmd]: http status code OK 17:25:05 INFO [http_cmd]: http status code OK 17:25:05 DEBUG --- status code --- 17:25:05 DEBUG --- status code --- 17:25:05 INFO [http_cmd]: http status code OK 17:25:05 DEBUG --- status code --- 17:25:05 INFO [http_cmd]: http status code OK 17:25:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:05 INFO [http_cmd]: http status code OK 17:25:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:05 INFO [http_cmd]: http status code OK 17:25:05 DEBUG --- status code --- 17:25:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:05 DEBUG --- status code --- 17:25:05 DEBUG --- http response --- 17:25:05 DEBUG --- status code --- 17:25:05 DEBUG --- http response --- 17:25:05 DEBUG --- status code --- 17:25:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:05 INFO [http_cmd]: http status code OK 17:25:05 INFO [http_cmd]: http status code OK 17:25:05 DEBUG --- http response --- 17:25:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:25:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:25:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:05 DEBUG --- http response --- 17:25:05 DEBUG --- status code --- 17:25:05 DEBUG --- status code --- 17:25:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:25:05 DEBUG --- http response --- 17:25:05 DEBUG ----- output ----- 17:25:05 DEBUG --- http response --- 17:25:05 DEBUG ----- output ----- 17:25:05 DEBUG --- http response --- 17:25:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:25:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:05 DEBUG --- http response --- 17:25:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:25:05 DEBUG { 17:25:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:25:05 DEBUG { 17:25:05 DEBUG ----- output ----- 17:25:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:25:05 DEBUG --- http response --- 17:25:05 DEBUG ----- output ----- 17:25:05 DEBUG ----- output ----- 17:25:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:25:05 DEBUG "status": "success", 17:25:05 DEBUG ----- output ----- 17:25:05 DEBUG "status": "success", 17:25:05 DEBUG { 17:25:05 DEBUG ----- output ----- 17:25:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:25:05 DEBUG { 17:25:05 DEBUG { 17:25:05 DEBUG ----- output ----- 17:25:05 DEBUG "data": { 17:25:05 DEBUG { 17:25:05 DEBUG "data": { 17:25:05 DEBUG "status": "success", 17:25:05 INFO [http_cmd]: http status code OK 17:25:05 DEBUG { 17:25:05 DEBUG ----- output ----- 17:25:05 DEBUG "status": "success", 17:25:05 DEBUG "status": "success", 17:25:05 DEBUG { 17:25:05 DEBUG "resultType": "vector", 17:25:05 DEBUG "status": "success", 17:25:05 DEBUG "resultType": "vector", 17:25:05 DEBUG "data": { 17:25:05 DEBUG --- status code --- 17:25:05 DEBUG "status": "success", 17:25:05 DEBUG { 17:25:05 DEBUG "data": { 17:25:05 DEBUG "data": { 17:25:05 DEBUG "status": "success", 17:25:05 DEBUG "result": [ 17:25:05 DEBUG "data": { 17:25:05 DEBUG "result": [ 17:25:05 DEBUG "resultType": "vector", 17:25:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:25:05 DEBUG "data": { 17:25:05 DEBUG "status": "success", 17:25:05 DEBUG "resultType": "vector", 17:25:05 DEBUG "resultType": "vector", 17:25:05 DEBUG "data": { 17:25:05 DEBUG { 17:25:05 DEBUG "resultType": "vector", 17:25:05 DEBUG { 17:25:05 DEBUG "result": [ 17:25:05 DEBUG --- http response --- 17:25:05 DEBUG "resultType": "vector", 17:25:05 DEBUG "data": { 17:25:05 DEBUG "result": [ 17:25:05 DEBUG "result": [ 17:25:05 DEBUG "resultType": "vector", 17:25:05 DEBUG "metric": { 17:25:05 DEBUG "result": [ 17:25:05 DEBUG "metric": { 17:25:05 DEBUG { 17:25:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:25:05 DEBUG "result": [ 17:25:05 DEBUG "resultType": "vector", 17:25:05 DEBUG "result": [ 17:25:05 DEBUG { 17:25:05 DEBUG "result": [ 17:25:05 DEBUG "instance": "10.142.0.107:9100" 17:25:05 DEBUG { 17:25:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:25:05 DEBUG "metric": { 17:25:05 DEBUG ----- output ----- 17:25:05 DEBUG { 17:25:05 DEBUG { 17:25:05 DEBUG { 17:25:05 DEBUG "metric": { 17:25:05 DEBUG { 17:25:05 DEBUG }, 17:25:05 DEBUG "metric": { 17:25:05 DEBUG }, 17:25:05 DEBUG "pod": "ds-cts-0" 17:25:05 DEBUG { 17:25:05 DEBUG "metric": { 17:25:05 DEBUG "metric": { 17:25:05 DEBUG "metric": { 17:25:05 DEBUG "pod": "ds-idrepo-1" 17:25:05 DEBUG "metric": { 17:25:05 DEBUG "value": [ 17:25:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:25:05 DEBUG "value": [ 17:25:05 DEBUG }, 17:25:05 DEBUG "status": "success", 17:25:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:25:05 DEBUG "pod": "ds-idrepo-0" 17:25:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:25:05 DEBUG }, 17:25:05 DEBUG "value": [ 17:25:05 DEBUG 17:25:05 DEBUG }, 17:25:05 DEBUG 17:25:05 DEBUG "value": [ 17:25:05 DEBUG "data": { 17:25:05 DEBUG }, 17:25:05 DEBUG }, 17:25:05 DEBUG }, 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG "value": [ 17:25:05 DEBUG "instance": "10.142.0.107:9100" 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG "resultType": "vector", 17:25:05 DEBUG "value": [ 17:25:05 DEBUG "value": [ 17:25:05 DEBUG "value": [ 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG }, 17:25:05 DEBUG "value": [ 17:25:05 DEBUG 17:25:05 DEBUG "result": [ 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG [...] 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG { 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG [...] 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG [...] 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG "metric": { 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG [...] 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG [...] 17:25:05 DEBUG 17:25:05 DEBUG [...] 17:25:05 DEBUG [...] 17:25:05 DEBUG "pod": "ds-idrepo-1" 17:25:05 DEBUG [...] 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG }, 17:25:05 DEBUG 17:25:05 DEBUG [...] 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG "instance": "10.142.0.43:9100" 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG "value": [ 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:25:05 DEBUG }, 17:25:05 DEBUG 17:25:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG 17:25:05 DEBUG "pod": "ds-idrepo-2" 17:25:05 DEBUG }, 17:25:05 INFO 17:25:05 DEBUG "value": [ 17:25:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:25:05 DEBUG }, 17:25:05 DEBUG "instance": "10.142.0.43:9100" 17:25:05 DEBUG "pod": "ds-cts-2" 17:25:05 DEBUG 17:25:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:25:05 DEBUG 17:25:05 DEBUG }, 17:25:05 DEBUG "value": [ 17:25:05 INFO [loop_until]: kubectl --namespace=xlou top pods 17:25:05 DEBUG 1713543905, 17:25:05 DEBUG }, 17:25:05 DEBUG "value": [ 17:25:05 DEBUG }, 17:25:05 DEBUG }, 17:25:05 DEBUG 17:25:05 DEBUG }, 17:25:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:25:05 DEBUG "value": [ 17:25:05 DEBUG 1713543905, 17:25:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:25:05 DEBUG "0.06856666666662174" 17:25:05 DEBUG "value": [ 17:25:05 DEBUG 1713543905, 17:25:05 DEBUG "value": [ 17:25:05 DEBUG "value": [ 17:25:05 INFO 17:25:05 DEBUG [...] 17:25:05 DEBUG "value": [ 17:25:05 DEBUG }, 17:25:05 DEBUG 1713543905, 17:25:05 DEBUG "0.0013547278499656612" 17:25:05 DEBUG ] 17:25:05 DEBUG 1713543905, 17:25:05 DEBUG "7.844079760787834" 17:25:05 DEBUG 1713543905, 17:25:05 DEBUG 1713543905, 17:25:05 INFO [loop_until]: kubectl --namespace=xlou top node 17:25:05 DEBUG 17:25:05 DEBUG 1713543905, 17:25:05 DEBUG "value": [ 17:25:05 DEBUG "0" 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG "2335.0333333333333" 17:25:05 DEBUG ] 17:25:05 DEBUG "56558314.53333333" 17:25:05 DEBUG "3.533333333333333" 17:25:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:25:05 DEBUG 17:25:05 DEBUG "2335.2" 17:25:05 DEBUG 1713543905, 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG ] 17:25:05 DEBUG 17:25:05 DEBUG ] 17:25:05 DEBUG "0.023" 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 DEBUG }, 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG "value": [ 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 DEBUG 1713543905, 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 DEBUG "0" 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG ] 17:25:05 DEBUG } 17:25:05 DEBUG } 17:25:05 INFO [loop_until]: OK (rc = 0) 17:25:05 DEBUG --- stdout --- 17:25:05 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11382m 8730Mi am-6fd554f8f7-btd4s 11383m 8777Mi am-6fd554f8f7-wxcsj 12217m 8783Mi ds-cts-0 9827m 12439Mi ds-cts-1 9723m 12377Mi ds-cts-2 10040m 12423Mi ds-idrepo-0 5738m 3420Mi ds-idrepo-1 5492m 3372Mi ds-idrepo-2 5736m 3424Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7951m 857Mi 17:25:05 DEBUG --- stderr --- 17:25:05 DEBUG 17:25:05 INFO [loop_until]: OK (rc = 0) 17:25:05 DEBUG --- stdout --- 17:25:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6179Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5532m 18% 6985Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 49m 0% 2689Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11283m 37% 12005Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9725m 32% 15454Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5215Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11678m 39% 11655Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12598m 42% 2970Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13411m 44% 10984Mi 9% gke-xlou-cdm-ds-0f601751-j488 6329m 21% 6658Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9912m 33% 17182Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6253m 20% 7196Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 10037m 33% 14895Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8108m 51% 4635Mi 7% 17:25:05 DEBUG --- stderr --- 17:25:05 DEBUG 17:26:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713543963" 17:26:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543963" 17:26:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543963" 17:26:03 ERROR [http_cmd]: ERROR 17:26:03 INFO [http_cmd]: http status code OK 17:26:03 ERROR --- status code --- 17:26:03 DEBUG --- status code --- 17:26:03 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:26:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:03 ERROR --- http response --- 17:26:03 DEBUG --- http response --- 17:26:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:26:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:26:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:26:03 INFO [http_cmd]: http status code OK 17:26:03 DEBUG --- status code --- 17:26:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:03 DEBUG --- http response --- 17:26:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:26:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713543963" 17:26:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543963" 17:26:03 INFO [http_cmd]: http status code OK 17:26:03 DEBUG --- status code --- 17:26:03 INFO [http_cmd]: http status code OK 17:26:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:03 DEBUG --- status code --- 17:26:03 DEBUG --- http response --- 17:26:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:26:03 DEBUG --- http response --- 17:26:03 DEBUG ----- output ----- 17:26:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:26:03 DEBUG { 17:26:03 DEBUG ----- output ----- 17:26:03 DEBUG "status": "success", 17:26:03 DEBUG { 17:26:03 DEBUG "data": { 17:26:03 DEBUG "status": "success", 17:26:03 DEBUG "resultType": "vector", 17:26:03 DEBUG "data": { 17:26:03 DEBUG "result": [ 17:26:03 DEBUG "resultType": "vector", 17:26:03 DEBUG { 17:26:03 DEBUG "result": [ 17:26:03 DEBUG "metric": { 17:26:03 DEBUG { 17:26:03 DEBUG "pod": "ds-idrepo-0" 17:26:03 DEBUG "metric": { 17:26:03 DEBUG }, 17:26:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:26:03 DEBUG "value": [ 17:26:03 DEBUG }, 17:26:03 DEBUG 17:26:03 DEBUG "value": [ 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG [...] 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG [...] 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG "pod": "ds-cts-2" 17:26:03 DEBUG 17:26:03 DEBUG }, 17:26:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:26:03 DEBUG "value": [ 17:26:03 DEBUG }, 17:26:03 DEBUG 1713543963, 17:26:03 DEBUG "value": [ 17:26:03 DEBUG "0" 17:26:03 DEBUG 1713543963, 17:26:03 DEBUG ] 17:26:03 DEBUG "2336.2666666666664" 17:26:03 DEBUG } 17:26:03 DEBUG ] 17:26:03 DEBUG ] 17:26:03 DEBUG } 17:26:03 DEBUG } 17:26:03 DEBUG ] 17:26:03 DEBUG } 17:26:03 DEBUG } 17:26:03 DEBUG } 17:26:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543963" 17:26:03 INFO [http_cmd]: http status code OK 17:26:03 DEBUG --- status code --- 17:26:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:03 DEBUG --- http response --- 17:26:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:26:03 DEBUG ----- output ----- 17:26:03 DEBUG { 17:26:03 DEBUG "status": "success", 17:26:03 DEBUG "data": { 17:26:03 DEBUG "resultType": "vector", 17:26:03 DEBUG "result": [ 17:26:03 DEBUG { 17:26:03 DEBUG "metric": { 17:26:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:26:03 DEBUG }, 17:26:03 DEBUG "value": [ 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG [...] 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:26:03 DEBUG }, 17:26:03 DEBUG "value": [ 17:26:03 DEBUG 1713543963, 17:26:03 DEBUG "0" 17:26:03 DEBUG ] 17:26:03 DEBUG } 17:26:03 DEBUG ] 17:26:03 DEBUG } 17:26:03 DEBUG } 17:26:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543963" 17:26:03 INFO [http_cmd]: http status code OK 17:26:03 DEBUG --- status code --- 17:26:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:03 DEBUG --- http response --- 17:26:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 17:26:03 DEBUG ----- output ----- 17:26:03 DEBUG { 17:26:03 DEBUG "status": "success", 17:26:03 DEBUG "data": { 17:26:03 DEBUG "resultType": "vector", 17:26:03 DEBUG "result": [ 17:26:03 DEBUG { 17:26:03 DEBUG "metric": { 17:26:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:26:03 DEBUG }, 17:26:03 DEBUG "value": [ 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG [...] 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:26:03 DEBUG }, 17:26:03 DEBUG "value": [ 17:26:03 DEBUG 1713543963, 17:26:03 DEBUG "0" 17:26:03 DEBUG ] 17:26:03 DEBUG } 17:26:03 DEBUG ] 17:26:03 DEBUG } 17:26:03 DEBUG } 17:26:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543963" 17:26:03 INFO [http_cmd]: http status code OK 17:26:03 DEBUG --- status code --- 17:26:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:03 DEBUG --- http response --- 17:26:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:26:03 DEBUG ----- output ----- 17:26:03 DEBUG { 17:26:03 DEBUG "status": "success", 17:26:03 DEBUG "data": { 17:26:03 DEBUG "resultType": "vector", 17:26:03 DEBUG "result": [ 17:26:03 DEBUG { 17:26:03 DEBUG "metric": { 17:26:03 DEBUG "instance": "10.142.0.107:9100" 17:26:03 DEBUG }, 17:26:03 DEBUG "value": [ 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG [...] 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG 17:26:03 DEBUG "instance": "10.142.0.43:9100" 17:26:03 DEBUG }, 17:26:03 DEBUG "value": [ 17:26:03 DEBUG 1713543963, 17:26:03 DEBUG "0.04533333333332999" 17:26:03 DEBUG ] 17:26:03 DEBUG } 17:26:03 DEBUG ] 17:26:03 DEBUG } 17:26:03 DEBUG } 17:26:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543964" 17:26:04 INFO [http_cmd]: http status code OK 17:26:04 DEBUG --- status code --- 17:26:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:04 DEBUG --- http response --- 17:26:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:26:04 DEBUG ----- output ----- 17:26:04 DEBUG { 17:26:04 DEBUG "status": "success", 17:26:04 DEBUG "data": { 17:26:04 DEBUG "resultType": "vector", 17:26:04 DEBUG "result": [ 17:26:04 DEBUG { 17:26:04 DEBUG "metric": { 17:26:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:26:04 DEBUG }, 17:26:04 DEBUG "value": [ 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG [...] 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:26:04 DEBUG }, 17:26:04 DEBUG "value": [ 17:26:04 DEBUG 1713543964, 17:26:04 DEBUG "3605816" 17:26:04 DEBUG ] 17:26:04 DEBUG } 17:26:04 DEBUG ] 17:26:04 DEBUG } 17:26:04 DEBUG } 17:26:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543964" 17:26:04 INFO [http_cmd]: http status code OK 17:26:04 DEBUG --- status code --- 17:26:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:04 DEBUG --- http response --- 17:26:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:26:04 DEBUG ----- output ----- 17:26:04 DEBUG { 17:26:04 DEBUG "status": "success", 17:26:04 DEBUG "data": { 17:26:04 DEBUG "resultType": "vector", 17:26:04 DEBUG "result": [ 17:26:04 DEBUG { 17:26:04 DEBUG "metric": { 17:26:04 DEBUG "instance": "10.142.0.107:9100" 17:26:04 DEBUG }, 17:26:04 DEBUG "value": [ 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG [...] 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG "instance": "10.142.0.43:9100" 17:26:04 DEBUG }, 17:26:04 DEBUG "value": [ 17:26:04 DEBUG 1713543964, 17:26:04 DEBUG "1697211083776" 17:26:04 DEBUG ] 17:26:04 DEBUG } 17:26:04 DEBUG ] 17:26:04 DEBUG } 17:26:04 DEBUG } 17:26:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543964" 17:26:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713543964" 17:26:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543964" 17:26:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543964" 17:26:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543964" 17:26:04 INFO [http_cmd]: http status code OK 17:26:04 DEBUG --- status code --- 17:26:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:04 DEBUG --- http response --- 17:26:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:26:04 DEBUG ----- output ----- 17:26:04 DEBUG { 17:26:04 DEBUG "status": "success", 17:26:04 DEBUG "data": { 17:26:04 DEBUG "resultType": "vector", 17:26:04 DEBUG "result": [ 17:26:04 DEBUG { 17:26:04 DEBUG "metric": { 17:26:04 DEBUG "pod": "ds-idrepo-0" 17:26:04 DEBUG }, 17:26:04 DEBUG "value": [ 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG [...] 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG "pod": "ds-idrepo-2" 17:26:04 DEBUG }, 17:26:04 DEBUG "value": [ 17:26:04 DEBUG 1713543964, 17:26:04 DEBUG "0" 17:26:04 DEBUG ] 17:26:04 DEBUG } 17:26:04 DEBUG ] 17:26:04 DEBUG } 17:26:04 DEBUG } 17:26:04 INFO [http_cmd]: http status code OK 17:26:04 DEBUG --- status code --- 17:26:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:04 DEBUG --- http response --- 17:26:04 INFO [http_cmd]: http status code OK 17:26:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:26:04 DEBUG --- status code --- 17:26:04 DEBUG ----- output ----- 17:26:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:04 DEBUG { 17:26:04 DEBUG --- http response --- 17:26:04 DEBUG "status": "success", 17:26:04 DEBUG "data": { 17:26:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:26:04 DEBUG "resultType": "vector", 17:26:04 DEBUG ----- output ----- 17:26:04 DEBUG "result": [ 17:26:04 DEBUG { 17:26:04 DEBUG { 17:26:04 DEBUG "status": "success", 17:26:04 DEBUG "metric": { 17:26:04 DEBUG "data": { 17:26:04 DEBUG "pod": "ds-idrepo-0" 17:26:04 DEBUG "resultType": "vector", 17:26:04 DEBUG }, 17:26:04 INFO [http_cmd]: http status code OK 17:26:04 INFO [http_cmd]: http status code OK 17:26:04 DEBUG "result": [ 17:26:04 DEBUG "value": [ 17:26:04 DEBUG --- status code --- 17:26:04 DEBUG --- status code --- 17:26:04 DEBUG { 17:26:04 DEBUG 17:26:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:04 DEBUG "metric": { 17:26:04 DEBUG 17:26:04 DEBUG --- http response --- 17:26:04 DEBUG --- http response --- 17:26:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:26:04 DEBUG 17:26:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:26:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:26:04 DEBUG }, 17:26:04 DEBUG [...] 17:26:04 DEBUG ----- output ----- 17:26:04 DEBUG ----- output ----- 17:26:04 DEBUG "value": [ 17:26:04 DEBUG 17:26:04 DEBUG { 17:26:04 DEBUG { 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG "status": "success", 17:26:04 DEBUG "status": "success", 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG "data": { 17:26:04 DEBUG "data": { 17:26:04 DEBUG 17:26:04 DEBUG "pod": "ds-idrepo-2" 17:26:04 DEBUG "resultType": "vector", 17:26:04 DEBUG "resultType": "vector", 17:26:04 DEBUG [...] 17:26:04 DEBUG }, 17:26:04 DEBUG "result": [ 17:26:04 DEBUG "result": [ 17:26:04 DEBUG { 17:26:04 DEBUG "metric": { 17:26:04 DEBUG { 17:26:04 DEBUG 17:26:04 DEBUG "value": [ 17:26:04 DEBUG "pod": "ds-cts-0" 17:26:04 DEBUG "metric": { 17:26:04 DEBUG 17:26:04 DEBUG 1713543964, 17:26:04 DEBUG }, 17:26:04 DEBUG "pod": "ds-cts-0" 17:26:04 DEBUG 17:26:04 DEBUG "0" 17:26:04 DEBUG "value": [ 17:26:04 DEBUG }, 17:26:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:26:04 DEBUG ] 17:26:04 DEBUG 17:26:04 DEBUG "value": [ 17:26:04 DEBUG }, 17:26:04 DEBUG } 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG "value": [ 17:26:04 DEBUG ] 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG 1713543964, 17:26:04 DEBUG } 17:26:04 DEBUG [...] 17:26:04 DEBUG 17:26:04 DEBUG "998801408" 17:26:04 DEBUG } 17:26:04 DEBUG 17:26:04 DEBUG 17:26:04 DEBUG ] 17:26:04 DEBUG [...] 17:26:04 DEBUG 17:26:04 DEBUG } 17:26:04 DEBUG 17:26:04 DEBUG "pod": "ds-cts-2" 17:26:04 DEBUG ] 17:26:04 DEBUG 17:26:04 DEBUG }, 17:26:04 DEBUG } 17:26:04 DEBUG 17:26:04 DEBUG "value": [ 17:26:04 DEBUG } 17:26:04 DEBUG "pod": "ds-cts-2" 17:26:04 DEBUG 1713543964, 17:26:04 DEBUG }, 17:26:04 DEBUG "0.000033333333333333335" 17:26:04 DEBUG "value": [ 17:26:04 DEBUG ] 17:26:04 DEBUG 1713543964, 17:26:04 DEBUG } 17:26:04 DEBUG "0" 17:26:04 DEBUG ] 17:26:04 DEBUG ] 17:26:04 DEBUG } 17:26:04 DEBUG } 17:26:04 DEBUG } 17:26:04 DEBUG ] 17:26:04 DEBUG } 17:26:04 DEBUG } 17:26:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543965" 17:26:05 INFO [http_cmd]: http status code OK 17:26:05 DEBUG --- status code --- 17:26:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:05 DEBUG --- http response --- 17:26:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:26:05 DEBUG ----- output ----- 17:26:05 DEBUG { 17:26:05 DEBUG "status": "success", 17:26:05 DEBUG "data": { 17:26:05 DEBUG "resultType": "vector", 17:26:05 DEBUG "result": [ 17:26:05 DEBUG { 17:26:05 DEBUG "metric": { 17:26:05 DEBUG "instance": "10.142.0.107:9100" 17:26:05 DEBUG }, 17:26:05 DEBUG "value": [ 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG [...] 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG "instance": "10.142.0.43:9100" 17:26:05 DEBUG }, 17:26:05 DEBUG "value": [ 17:26:05 DEBUG 1713543965, 17:26:05 DEBUG "56657140.233333334" 17:26:05 DEBUG ] 17:26:05 DEBUG } 17:26:05 DEBUG ] 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543965" 17:26:05 INFO [http_cmd]: http status code OK 17:26:05 DEBUG --- status code --- 17:26:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:05 DEBUG --- http response --- 17:26:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:26:05 DEBUG ----- output ----- 17:26:05 DEBUG { 17:26:05 DEBUG "status": "success", 17:26:05 DEBUG "data": { 17:26:05 DEBUG "resultType": "vector", 17:26:05 DEBUG "result": [ 17:26:05 DEBUG { 17:26:05 DEBUG "metric": { 17:26:05 DEBUG "pod": "ds-idrepo-0" 17:26:05 DEBUG }, 17:26:05 DEBUG "value": [ 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG [...] 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:26:05 DEBUG }, 17:26:05 DEBUG "value": [ 17:26:05 DEBUG 1713543965, 17:26:05 DEBUG "0" 17:26:05 DEBUG ] 17:26:05 DEBUG } 17:26:05 DEBUG ] 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543965" 17:26:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543965" 17:26:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713543965" 17:26:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543965" 17:26:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543965" 17:26:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713543965" 17:26:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543965" 17:26:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543965" 17:26:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713543965" 17:26:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713543965" 17:26:05 INFO [http_cmd]: http status code OK 17:26:05 DEBUG --- status code --- 17:26:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:05 INFO [http_cmd]: http status code OK 17:26:05 DEBUG --- http response --- 17:26:05 DEBUG --- status code --- 17:26:05 INFO [http_cmd]: http status code OK 17:26:05 INFO [http_cmd]: http status code OK 17:26:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:26:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:05 INFO [http_cmd]: http status code OK 17:26:05 DEBUG --- status code --- 17:26:05 INFO [http_cmd]: http status code OK 17:26:05 INFO [http_cmd]: http status code OK 17:26:05 INFO [http_cmd]: http status code OK 17:26:05 DEBUG --- status code --- 17:26:05 INFO [http_cmd]: http status code OK 17:26:05 DEBUG ----- output ----- 17:26:05 DEBUG --- http response --- 17:26:05 DEBUG --- status code --- 17:26:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:05 DEBUG --- status code --- 17:26:05 DEBUG --- status code --- 17:26:05 DEBUG --- status code --- 17:26:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:05 INFO [http_cmd]: http status code OK 17:26:05 DEBUG --- status code --- 17:26:05 DEBUG { 17:26:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:26:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:05 DEBUG --- http response --- 17:26:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:05 DEBUG --- http response --- 17:26:05 DEBUG --- status code --- 17:26:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:05 DEBUG "status": "success", 17:26:05 DEBUG ----- output ----- 17:26:05 DEBUG --- http response --- 17:26:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:26:05 DEBUG --- http response --- 17:26:05 DEBUG --- http response --- 17:26:05 DEBUG --- http response --- 17:26:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:26:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:26:05 DEBUG --- http response --- 17:26:05 DEBUG "data": { 17:26:05 DEBUG { 17:26:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:26:05 DEBUG ----- output ----- 17:26:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:26:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:26:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:26:05 DEBUG ----- output ----- 17:26:05 DEBUG --- http response --- 17:26:05 DEBUG "resultType": "vector", 17:26:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:26:05 DEBUG "status": "success", 17:26:05 DEBUG ----- output ----- 17:26:05 DEBUG { 17:26:05 DEBUG ----- output ----- 17:26:05 DEBUG ----- output ----- 17:26:05 DEBUG ----- output ----- 17:26:05 DEBUG { 17:26:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:26:05 DEBUG "result": [ 17:26:05 DEBUG ----- output ----- 17:26:05 DEBUG "data": { 17:26:05 DEBUG { 17:26:05 DEBUG "status": "success", 17:26:05 DEBUG { 17:26:05 DEBUG { 17:26:05 DEBUG { 17:26:05 DEBUG "status": "success", 17:26:05 DEBUG ----- output ----- 17:26:05 DEBUG { 17:26:05 DEBUG { 17:26:05 DEBUG "resultType": "vector", 17:26:05 DEBUG "status": "success", 17:26:05 DEBUG "data": { 17:26:05 DEBUG "status": "success", 17:26:05 DEBUG "status": "success", 17:26:05 DEBUG "status": "success", 17:26:05 DEBUG "data": { 17:26:05 DEBUG { 17:26:05 DEBUG "metric": { 17:26:05 DEBUG "status": "success", 17:26:05 DEBUG "result": [ 17:26:05 DEBUG "data": { 17:26:05 DEBUG "resultType": "vector", 17:26:05 DEBUG "data": { 17:26:05 DEBUG "data": { 17:26:05 DEBUG "data": { 17:26:05 DEBUG "resultType": "vector", 17:26:05 DEBUG "status": "success", 17:26:05 DEBUG "pod": "ds-idrepo-0" 17:26:05 DEBUG "data": { 17:26:05 DEBUG { 17:26:05 DEBUG "resultType": "vector", 17:26:05 DEBUG "result": [ 17:26:05 DEBUG "resultType": "vector", 17:26:05 DEBUG "resultType": "vector", 17:26:05 DEBUG "resultType": "vector", 17:26:05 DEBUG "result": [ 17:26:05 DEBUG "data": { 17:26:05 DEBUG }, 17:26:05 DEBUG "resultType": "vector", 17:26:05 DEBUG "metric": { 17:26:05 DEBUG "result": [ 17:26:05 DEBUG { 17:26:05 DEBUG "result": [ 17:26:05 DEBUG "result": [ 17:26:05 DEBUG "result": [ 17:26:05 DEBUG { 17:26:05 DEBUG "resultType": "vector", 17:26:05 DEBUG "value": [ 17:26:05 DEBUG "result": [ 17:26:05 DEBUG "instance": "10.142.0.107:9100" 17:26:05 DEBUG { 17:26:05 DEBUG "metric": { 17:26:05 DEBUG { 17:26:05 DEBUG { 17:26:05 DEBUG { 17:26:05 DEBUG "metric": { 17:26:05 DEBUG "result": [ 17:26:05 DEBUG 17:26:05 DEBUG { 17:26:05 DEBUG }, 17:26:05 DEBUG "metric": { 17:26:05 DEBUG "instance": "10.142.0.107:9100" 17:26:05 DEBUG "metric": { 17:26:05 DEBUG "metric": { 17:26:05 DEBUG "metric": { 17:26:05 DEBUG "pod": "ds-idrepo-0" 17:26:05 DEBUG { 17:26:05 DEBUG 17:26:05 DEBUG "metric": { 17:26:05 DEBUG "value": [ 17:26:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:26:05 DEBUG }, 17:26:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:26:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:26:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:26:05 DEBUG }, 17:26:05 DEBUG "metric": { 17:26:05 DEBUG 17:26:05 DEBUG "pod": "ds-idrepo-0" 17:26:05 DEBUG 17:26:05 DEBUG }, 17:26:05 DEBUG "value": [ 17:26:05 DEBUG }, 17:26:05 DEBUG }, 17:26:05 DEBUG }, 17:26:05 DEBUG "value": [ 17:26:05 DEBUG "pod": "ds-cts-0" 17:26:05 DEBUG [...] 17:26:05 DEBUG }, 17:26:05 DEBUG 17:26:05 DEBUG "value": [ 17:26:05 DEBUG 17:26:05 DEBUG "value": [ 17:26:05 DEBUG "value": [ 17:26:05 DEBUG "value": [ 17:26:05 DEBUG 17:26:05 DEBUG }, 17:26:05 DEBUG 17:26:05 DEBUG "value": [ 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG "value": [ 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG [...] 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG [...] 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG [...] 17:26:05 DEBUG [...] 17:26:05 DEBUG 17:26:05 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:26:05 DEBUG 17:26:05 DEBUG [...] 17:26:05 DEBUG [...] 17:26:05 DEBUG 17:26:05 DEBUG [...] 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG }, 17:26:05 DEBUG "value": [ 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG [...] 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG [...] 17:26:05 DEBUG 17:26:05 DEBUG 1713543965, 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG "0.000006519967880085049" 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG "instance": "10.142.0.43:9100" 17:26:05 DEBUG 17:26:05 DEBUG 17:26:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:26:05 DEBUG "pod": "ds-idrepo-2" 17:26:05 DEBUG 17:26:05 DEBUG "instance": "10.142.0.43:9100" 17:26:05 DEBUG ] 17:26:05 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:26:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:26:05 DEBUG }, 17:26:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:26:05 DEBUG 17:26:05 DEBUG }, 17:26:05 DEBUG }, 17:26:05 DEBUG 17:26:05 DEBUG }, 17:26:05 DEBUG } 17:26:05 DEBUG }, 17:26:05 DEBUG }, 17:26:05 DEBUG "value": [ 17:26:05 DEBUG }, 17:26:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:26:05 DEBUG "value": [ 17:26:05 DEBUG "value": [ 17:26:05 DEBUG "pod": "ds-cts-2" 17:26:05 DEBUG "value": [ 17:26:05 DEBUG ] 17:26:05 DEBUG "value": [ 17:26:05 DEBUG "value": [ 17:26:05 DEBUG 1713543965, 17:26:05 DEBUG "value": [ 17:26:05 DEBUG }, 17:26:05 DEBUG 1713543965, 17:26:05 DEBUG 1713543965, 17:26:05 DEBUG }, 17:26:05 DEBUG 1713543965, 17:26:05 DEBUG } 17:26:05 DEBUG 1713543965, 17:26:05 DEBUG 1713543965, 17:26:05 DEBUG "56649882.5" 17:26:05 DEBUG 1713543965, 17:26:05 DEBUG "value": [ 17:26:05 DEBUG "7.8917263319694015" 17:26:05 DEBUG "0" 17:26:05 DEBUG "value": [ 17:26:05 DEBUG "0.07396666666665794" 17:26:05 DEBUG } 17:26:05 DEBUG "0" 17:26:05 DEBUG "2336.0666666666666" 17:26:05 DEBUG ] 17:26:05 DEBUG "2336.133333333333" 17:26:05 DEBUG 1713543965, 17:26:05 DEBUG ] 17:26:05 DEBUG ] 17:26:05 DEBUG 1713543965, 17:26:05 DEBUG ] 17:26:05 DEBUG ] 17:26:05 DEBUG ] 17:26:05 DEBUG } 17:26:05 DEBUG ] 17:26:05 DEBUG "0.023" 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG ] 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG ] 17:26:05 DEBUG } 17:26:05 DEBUG ] 17:26:05 DEBUG ] 17:26:05 DEBUG "4.366666666666666" 17:26:05 DEBUG } 17:26:05 DEBUG ] 17:26:05 DEBUG ] 17:26:05 DEBUG ] 17:26:05 DEBUG } 17:26:05 DEBUG ] 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG ] 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG ] 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG ] 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 DEBUG } 17:26:05 INFO 17:26:05 INFO [loop_until]: kubectl --namespace=xlou top pods 17:26:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:26:05 INFO 17:26:05 INFO [loop_until]: kubectl --namespace=xlou top node 17:26:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:26:05 INFO [loop_until]: OK (rc = 0) 17:26:05 DEBUG --- stdout --- 17:26:05 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11592m 8732Mi am-6fd554f8f7-btd4s 11685m 8782Mi am-6fd554f8f7-wxcsj 12486m 8785Mi ds-cts-0 9489m 12440Mi ds-cts-1 9955m 12378Mi ds-cts-2 10129m 12423Mi ds-idrepo-0 5591m 3420Mi ds-idrepo-1 5521m 3379Mi ds-idrepo-2 5522m 3434Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8046m 859Mi 17:26:05 DEBUG --- stderr --- 17:26:05 DEBUG 17:26:05 INFO [loop_until]: OK (rc = 0) 17:26:05 DEBUG --- stdout --- 17:26:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6179Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5430m 18% 6986Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 46m 0% 2685Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11576m 38% 12001Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9496m 31% 15454Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5211Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11907m 39% 11654Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12373m 41% 2963Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13161m 44% 10990Mi 9% gke-xlou-cdm-ds-0f601751-j488 6277m 21% 6661Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9914m 33% 17191Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6254m 20% 7191Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9852m 32% 14902Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8052m 50% 4636Mi 7% 17:26:05 DEBUG --- stderr --- 17:26:05 DEBUG 17:27:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544023" 17:27:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544023" 17:27:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544023" 17:27:03 ERROR [http_cmd]: ERROR 17:27:03 ERROR --- status code --- 17:27:03 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:27:03 ERROR --- http response --- 17:27:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:27:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:27:03 INFO [http_cmd]: http status code OK 17:27:03 DEBUG --- status code --- 17:27:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:03 DEBUG --- http response --- 17:27:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:27:03 INFO [http_cmd]: http status code OK 17:27:03 DEBUG --- status code --- 17:27:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:03 DEBUG --- http response --- 17:27:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:27:03 DEBUG ----- output ----- 17:27:03 DEBUG { 17:27:03 DEBUG "status": "success", 17:27:03 DEBUG "data": { 17:27:03 DEBUG "resultType": "vector", 17:27:03 DEBUG "result": [ 17:27:03 DEBUG { 17:27:03 DEBUG "metric": { 17:27:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:27:03 DEBUG }, 17:27:03 DEBUG "value": [ 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG [...] 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:27:03 DEBUG }, 17:27:03 DEBUG "value": [ 17:27:03 DEBUG 1713544023, 17:27:03 DEBUG "0" 17:27:03 DEBUG ] 17:27:03 DEBUG } 17:27:03 DEBUG ] 17:27:03 DEBUG } 17:27:03 DEBUG } 17:27:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544023" 17:27:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544023" 17:27:03 INFO [http_cmd]: http status code OK 17:27:03 INFO [http_cmd]: http status code OK 17:27:03 DEBUG --- status code --- 17:27:03 DEBUG --- status code --- 17:27:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:03 DEBUG --- http response --- 17:27:03 DEBUG --- http response --- 17:27:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:27:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:27:03 DEBUG ----- output ----- 17:27:03 DEBUG ----- output ----- 17:27:03 DEBUG { 17:27:03 DEBUG { 17:27:03 DEBUG "status": "success", 17:27:03 DEBUG "status": "success", 17:27:03 DEBUG "data": { 17:27:03 DEBUG "data": { 17:27:03 DEBUG "resultType": "vector", 17:27:03 DEBUG "resultType": "vector", 17:27:03 DEBUG "result": [ 17:27:03 DEBUG "result": [ 17:27:03 DEBUG { 17:27:03 DEBUG { 17:27:03 DEBUG "metric": { 17:27:03 DEBUG "metric": { 17:27:03 DEBUG "pod": "ds-idrepo-0" 17:27:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:27:03 DEBUG }, 17:27:03 DEBUG }, 17:27:03 DEBUG "value": [ 17:27:03 DEBUG "value": [ 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG [...] 17:27:03 DEBUG [...] 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG "pod": "ds-cts-2" 17:27:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:27:03 DEBUG }, 17:27:03 DEBUG }, 17:27:03 DEBUG "value": [ 17:27:03 DEBUG "value": [ 17:27:03 DEBUG 1713544023, 17:27:03 DEBUG 1713544023, 17:27:03 DEBUG "0" 17:27:03 DEBUG "2341.133333333333" 17:27:03 DEBUG ] 17:27:03 DEBUG ] 17:27:03 DEBUG } 17:27:03 DEBUG } 17:27:03 DEBUG ] 17:27:03 DEBUG ] 17:27:03 DEBUG } 17:27:03 DEBUG } 17:27:03 DEBUG } 17:27:03 DEBUG } 17:27:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544023" 17:27:03 INFO [http_cmd]: http status code OK 17:27:03 DEBUG --- status code --- 17:27:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:03 DEBUG --- http response --- 17:27:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:27:03 DEBUG ----- output ----- 17:27:03 DEBUG { 17:27:03 DEBUG "status": "success", 17:27:03 DEBUG "data": { 17:27:03 DEBUG "resultType": "vector", 17:27:03 DEBUG "result": [ 17:27:03 DEBUG { 17:27:03 DEBUG "metric": { 17:27:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:27:03 DEBUG }, 17:27:03 DEBUG "value": [ 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG [...] 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:27:03 DEBUG }, 17:27:03 DEBUG "value": [ 17:27:03 DEBUG 1713544023, 17:27:03 DEBUG "0" 17:27:03 DEBUG ] 17:27:03 DEBUG } 17:27:03 DEBUG ] 17:27:03 DEBUG } 17:27:03 DEBUG } 17:27:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544023" 17:27:03 INFO [http_cmd]: http status code OK 17:27:03 DEBUG --- status code --- 17:27:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:03 DEBUG --- http response --- 17:27:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:27:03 DEBUG ----- output ----- 17:27:03 DEBUG { 17:27:03 DEBUG "status": "success", 17:27:03 DEBUG "data": { 17:27:03 DEBUG "resultType": "vector", 17:27:03 DEBUG "result": [ 17:27:03 DEBUG { 17:27:03 DEBUG "metric": { 17:27:03 DEBUG "pod": "ds-idrepo-0" 17:27:03 DEBUG }, 17:27:03 DEBUG "value": [ 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG [...] 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:27:03 DEBUG }, 17:27:03 DEBUG "value": [ 17:27:03 DEBUG 1713544023, 17:27:03 DEBUG "0" 17:27:03 DEBUG ] 17:27:03 DEBUG } 17:27:03 DEBUG ] 17:27:03 DEBUG } 17:27:03 DEBUG } 17:27:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544023" 17:27:03 INFO [http_cmd]: http status code OK 17:27:03 DEBUG --- status code --- 17:27:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:03 DEBUG --- http response --- 17:27:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:27:03 DEBUG ----- output ----- 17:27:03 DEBUG { 17:27:03 DEBUG "status": "success", 17:27:03 DEBUG "data": { 17:27:03 DEBUG "resultType": "vector", 17:27:03 DEBUG "result": [ 17:27:03 DEBUG { 17:27:03 DEBUG "metric": { 17:27:03 DEBUG "instance": "10.142.0.107:9100" 17:27:03 DEBUG }, 17:27:03 DEBUG "value": [ 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG [...] 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG 17:27:03 DEBUG "instance": "10.142.0.43:9100" 17:27:03 DEBUG }, 17:27:03 DEBUG "value": [ 17:27:03 DEBUG 1713544023, 17:27:03 DEBUG "0.04666666666667159" 17:27:03 DEBUG ] 17:27:03 DEBUG } 17:27:03 DEBUG ] 17:27:03 DEBUG } 17:27:03 DEBUG } 17:27:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544024" 17:27:04 INFO [http_cmd]: http status code OK 17:27:04 DEBUG --- status code --- 17:27:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:04 DEBUG --- http response --- 17:27:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:27:04 DEBUG ----- output ----- 17:27:04 DEBUG { 17:27:04 DEBUG "status": "success", 17:27:04 DEBUG "data": { 17:27:04 DEBUG "resultType": "vector", 17:27:04 DEBUG "result": [ 17:27:04 DEBUG { 17:27:04 DEBUG "metric": { 17:27:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:27:04 DEBUG }, 17:27:04 DEBUG "value": [ 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG [...] 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:27:04 DEBUG }, 17:27:04 DEBUG "value": [ 17:27:04 DEBUG 1713544024, 17:27:04 DEBUG "3745998" 17:27:04 DEBUG ] 17:27:04 DEBUG } 17:27:04 DEBUG ] 17:27:04 DEBUG } 17:27:04 DEBUG } 17:27:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544024" 17:27:04 INFO [http_cmd]: http status code OK 17:27:04 DEBUG --- status code --- 17:27:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:04 DEBUG --- http response --- 17:27:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:27:04 DEBUG ----- output ----- 17:27:04 DEBUG { 17:27:04 DEBUG "status": "success", 17:27:04 DEBUG "data": { 17:27:04 DEBUG "resultType": "vector", 17:27:04 DEBUG "result": [ 17:27:04 DEBUG { 17:27:04 DEBUG "metric": { 17:27:04 DEBUG "instance": "10.142.0.107:9100" 17:27:04 DEBUG }, 17:27:04 DEBUG "value": [ 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG [...] 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG "instance": "10.142.0.43:9100" 17:27:04 DEBUG }, 17:27:04 DEBUG "value": [ 17:27:04 DEBUG 1713544024, 17:27:04 DEBUG "1700063116288" 17:27:04 DEBUG ] 17:27:04 DEBUG } 17:27:04 DEBUG ] 17:27:04 DEBUG } 17:27:04 DEBUG } 17:27:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544024" 17:27:04 INFO [http_cmd]: http status code OK 17:27:04 DEBUG --- status code --- 17:27:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:04 DEBUG --- http response --- 17:27:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:27:04 DEBUG ----- output ----- 17:27:04 DEBUG { 17:27:04 DEBUG "status": "success", 17:27:04 DEBUG "data": { 17:27:04 DEBUG "resultType": "vector", 17:27:04 DEBUG "result": [ 17:27:04 DEBUG { 17:27:04 DEBUG "metric": { 17:27:04 DEBUG "pod": "ds-idrepo-0" 17:27:04 DEBUG }, 17:27:04 DEBUG "value": [ 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG [...] 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG "pod": "ds-idrepo-2" 17:27:04 DEBUG }, 17:27:04 DEBUG "value": [ 17:27:04 DEBUG 1713544024, 17:27:04 DEBUG "0" 17:27:04 DEBUG ] 17:27:04 DEBUG } 17:27:04 DEBUG ] 17:27:04 DEBUG } 17:27:04 DEBUG } 17:27:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544024" 17:27:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544024" 17:27:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544024" 17:27:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544024" 17:27:04 INFO [http_cmd]: http status code OK 17:27:04 DEBUG --- status code --- 17:27:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:04 DEBUG --- http response --- 17:27:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:27:04 DEBUG ----- output ----- 17:27:04 DEBUG { 17:27:04 DEBUG "status": "success", 17:27:04 DEBUG "data": { 17:27:04 DEBUG "resultType": "vector", 17:27:04 DEBUG "result": [ 17:27:04 DEBUG { 17:27:04 DEBUG "metric": { 17:27:04 DEBUG "pod": "ds-idrepo-0" 17:27:04 DEBUG }, 17:27:04 DEBUG "value": [ 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG [...] 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG 17:27:04 DEBUG "pod": "ds-idrepo-2" 17:27:04 DEBUG }, 17:27:04 DEBUG "value": [ 17:27:04 DEBUG 1713544024, 17:27:04 DEBUG "0" 17:27:04 DEBUG ] 17:27:04 DEBUG } 17:27:04 DEBUG ] 17:27:04 DEBUG } 17:27:04 DEBUG } 17:27:04 INFO [http_cmd]: http status code OK 17:27:04 DEBUG --- status code --- 17:27:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:04 DEBUG --- http response --- 17:27:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:27:04 DEBUG ----- output ----- 17:27:04 DEBUG { 17:27:04 DEBUG "status": "success", 17:27:04 DEBUG "data": { 17:27:04 DEBUG "resultType": "vector", 17:27:04 DEBUG "result": [ 17:27:04 DEBUG { 17:27:04 DEBUG "metric": { 17:27:04 DEBUG "pod": "ds-cts-0" 17:27:04 DEBUG }, 17:27:04 INFO [http_cmd]: http status code OK 17:27:04 DEBUG "value": [ 17:27:04 DEBUG --- status code --- 17:27:04 DEBUG 17:27:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:04 DEBUG 17:27:04 DEBUG --- http response --- 17:27:04 DEBUG 17:27:04 DEBUG [...] 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:27:05 DEBUG 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG 17:27:05 DEBUG --- status code --- 17:27:05 DEBUG { 17:27:05 DEBUG 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG "pod": "ds-cts-2" 17:27:05 DEBUG --- http response --- 17:27:05 DEBUG "data": { 17:27:05 DEBUG }, 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG "value": [ 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG "result": [ 17:27:05 DEBUG 1713544024, 17:27:05 DEBUG { 17:27:05 DEBUG { 17:27:05 DEBUG "0" 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG "metric": { 17:27:05 DEBUG ] 17:27:05 DEBUG "data": { 17:27:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:27:05 DEBUG } 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG }, 17:27:05 DEBUG ] 17:27:05 DEBUG "result": [ 17:27:05 DEBUG "value": [ 17:27:05 DEBUG } 17:27:05 DEBUG { 17:27:05 DEBUG 17:27:05 DEBUG } 17:27:05 DEBUG "metric": { 17:27:05 DEBUG 17:27:05 DEBUG "pod": "ds-cts-0" 17:27:05 DEBUG 17:27:05 DEBUG }, 17:27:05 DEBUG [...] 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:27:05 DEBUG [...] 17:27:05 DEBUG }, 17:27:05 DEBUG 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 17:27:05 DEBUG 1713544024, 17:27:05 DEBUG 17:27:05 DEBUG "1000153088" 17:27:05 DEBUG "pod": "ds-cts-2" 17:27:05 DEBUG ] 17:27:05 DEBUG }, 17:27:05 DEBUG } 17:27:05 DEBUG "value": [ 17:27:05 DEBUG ] 17:27:05 DEBUG 1713544024, 17:27:05 DEBUG } 17:27:05 DEBUG "0.00002266" 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544025" 17:27:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544025" 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 DEBUG --- status code --- 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG --- http response --- 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG { 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG "data": { 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG "result": [ 17:27:05 DEBUG { 17:27:05 DEBUG "metric": { 17:27:05 DEBUG "instance": "10.142.0.107:9100" 17:27:05 DEBUG }, 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG [...] 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG "instance": "10.142.0.43:9100" 17:27:05 DEBUG }, 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 1713544025, 17:27:05 DEBUG "56781278.7" 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 DEBUG --- status code --- 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG --- http response --- 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG { 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG "data": { 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG "result": [ 17:27:05 DEBUG { 17:27:05 DEBUG "metric": { 17:27:05 DEBUG "pod": "ds-idrepo-0" 17:27:05 DEBUG }, 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG [...] 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG "pod": "idm-b85f9d978-txpwv" 17:27:05 DEBUG }, 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 1713544025, 17:27:05 DEBUG "571.8673647469459" 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544025" 17:27:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544025" 17:27:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544025" 17:27:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544025" 17:27:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544025" 17:27:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544025" 17:27:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544025" 17:27:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544025" 17:27:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544025" 17:27:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544025" 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 DEBUG --- status code --- 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 DEBUG --- status code --- 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG --- status code --- 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG --- status code --- 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 DEBUG --- http response --- 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG --- status code --- 17:27:05 DEBUG --- http response --- 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG --- status code --- 17:27:05 DEBUG --- status code --- 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:27:05 DEBUG --- http response --- 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG --- http response --- 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG --- status code --- 17:27:05 DEBUG --- http response --- 17:27:05 INFO [http_cmd]: http status code OK 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:27:05 DEBUG --- http response --- 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG --- status code --- 17:27:05 DEBUG --- http response --- 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG --- status code --- 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:27:05 DEBUG { 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG { 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:27:05 DEBUG --- http response --- 17:27:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG { 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG { 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG --- http response --- 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG --- http response --- 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:27:05 DEBUG { 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG "data": { 17:27:05 DEBUG { 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG "data": { 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:27:05 DEBUG { 17:27:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG "data": { 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG "data": { 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG ----- output ----- 17:27:05 DEBUG { 17:27:05 DEBUG "data": { 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG "result": [ 17:27:05 DEBUG "data": { 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG "result": [ 17:27:05 DEBUG { 17:27:05 DEBUG "data": { 17:27:05 DEBUG { 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG "result": [ 17:27:05 DEBUG { 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG "result": [ 17:27:05 DEBUG { 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG "status": "success", 17:27:05 DEBUG "data": { 17:27:05 DEBUG "result": [ 17:27:05 DEBUG { 17:27:05 DEBUG "metric": { 17:27:05 DEBUG "result": [ 17:27:05 DEBUG { 17:27:05 DEBUG "metric": { 17:27:05 DEBUG "data": { 17:27:05 DEBUG "result": [ 17:27:05 DEBUG "data": { 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG { 17:27:05 DEBUG "metric": { 17:27:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:27:05 DEBUG { 17:27:05 DEBUG "metric": { 17:27:05 DEBUG "pod": "ds-idrepo-0" 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG { 17:27:05 DEBUG "resultType": "vector", 17:27:05 DEBUG "result": [ 17:27:05 DEBUG "metric": { 17:27:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:27:05 DEBUG }, 17:27:05 DEBUG "metric": { 17:27:05 DEBUG "instance": "10.142.0.107:9100" 17:27:05 DEBUG }, 17:27:05 DEBUG "result": [ 17:27:05 DEBUG "metric": { 17:27:05 DEBUG "result": [ 17:27:05 DEBUG { 17:27:05 DEBUG "metric": { 17:27:05 DEBUG }, 17:27:05 DEBUG "value": [ 17:27:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:27:05 DEBUG }, 17:27:05 DEBUG "value": [ 17:27:05 DEBUG { 17:27:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:27:05 DEBUG { 17:27:05 DEBUG "pod": "ds-idrepo-0" 17:27:05 DEBUG "pod": "ds-idrepo-0" 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 17:27:05 DEBUG }, 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 17:27:05 DEBUG "metric": { 17:27:05 DEBUG }, 17:27:05 DEBUG "metric": { 17:27:05 DEBUG }, 17:27:05 DEBUG }, 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG "instance": "10.142.0.107:9100" 17:27:05 DEBUG "value": [ 17:27:05 DEBUG "pod": "ds-cts-0" 17:27:05 DEBUG "value": [ 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG }, 17:27:05 DEBUG 17:27:05 DEBUG }, 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG [...] 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG [...] 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG [...] 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG [...] 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG [...] 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG [...] 17:27:05 DEBUG 17:27:05 DEBUG [...] 17:27:05 DEBUG [...] 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:27:05 DEBUG [...] 17:27:05 DEBUG 17:27:05 DEBUG [...] 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:27:05 DEBUG }, 17:27:05 DEBUG 17:27:05 DEBUG "instance": "10.142.0.43:9100" 17:27:05 DEBUG }, 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG "pod": "ds-idrepo-2" 17:27:05 DEBUG }, 17:27:05 DEBUG "value": [ 17:27:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:27:05 DEBUG }, 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 17:27:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:27:05 DEBUG 17:27:05 DEBUG 17:27:05 DEBUG }, 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 1713544025, 17:27:05 DEBUG }, 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 1713544025, 17:27:05 DEBUG 17:27:05 DEBUG }, 17:27:05 DEBUG 17:27:05 DEBUG "pod": "idm-b85f9d978-txpwv" 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 1713544025, 17:27:05 DEBUG "7.884936998084698" 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 1713544025, 17:27:05 DEBUG "0" 17:27:05 DEBUG "instance": "10.142.0.43:9100" 17:27:05 DEBUG "value": [ 17:27:05 DEBUG "pod": "ds-cts-2" 17:27:05 DEBUG }, 17:27:05 DEBUG 1713544025, 17:27:05 DEBUG "2340.9666666666667" 17:27:05 DEBUG ] 17:27:05 DEBUG 1713544025, 17:27:05 DEBUG "2341" 17:27:05 DEBUG ] 17:27:05 DEBUG }, 17:27:05 DEBUG 1713544025, 17:27:05 DEBUG }, 17:27:05 DEBUG "value": [ 17:27:05 DEBUG 1713544025, 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG "56773861.099999994" 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG "value": [ 17:27:05 DEBUG "0.023" 17:27:05 DEBUG ] 17:27:05 DEBUG "value": [ 17:27:05 DEBUG "0" 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG 1713544025, 17:27:05 DEBUG "0" 17:27:05 DEBUG } 17:27:05 DEBUG 1713544025, 17:27:05 DEBUG ] 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG "0.07209999999994068" 17:27:05 DEBUG ] 17:27:05 DEBUG ] 17:27:05 DEBUG "4.8" 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG ] 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 DEBUG } 17:27:05 INFO 17:27:05 INFO [loop_until]: kubectl --namespace=xlou top pods 17:27:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:27:05 INFO 17:27:05 INFO [loop_until]: kubectl --namespace=xlou top node 17:27:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:27:05 INFO [loop_until]: OK (rc = 0) 17:27:05 DEBUG --- stdout --- 17:27:05 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11604m 8747Mi am-6fd554f8f7-btd4s 11701m 8791Mi am-6fd554f8f7-wxcsj 12087m 8802Mi ds-cts-0 9516m 12441Mi ds-cts-1 10030m 12379Mi ds-cts-2 10160m 12425Mi ds-idrepo-0 5660m 3420Mi ds-idrepo-1 5439m 3387Mi ds-idrepo-2 5438m 3434Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7808m 861Mi 17:27:05 DEBUG --- stderr --- 17:27:05 DEBUG 17:27:05 INFO [loop_until]: OK (rc = 0) 17:27:05 DEBUG --- stdout --- 17:27:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6187Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5431m 18% 7000Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 46m 0% 2690Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11732m 39% 12028Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9743m 32% 15463Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5215Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11927m 39% 11666Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12392m 41% 2969Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13170m 44% 11021Mi 9% gke-xlou-cdm-ds-0f601751-j488 6043m 20% 6654Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9642m 32% 17189Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6020m 20% 7204Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 10047m 33% 14900Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8071m 50% 4631Mi 7% 17:27:05 DEBUG --- stderr --- 17:27:05 DEBUG 17:28:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544083" 17:28:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544083" 17:28:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544083" 17:28:03 ERROR [http_cmd]: ERROR 17:28:03 ERROR --- status code --- 17:28:03 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:28:03 ERROR --- http response --- 17:28:03 INFO [http_cmd]: http status code OK 17:28:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:28:03 DEBUG --- status code --- 17:28:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:28:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:03 DEBUG --- http response --- 17:28:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:28:03 INFO [http_cmd]: http status code OK 17:28:03 DEBUG --- status code --- 17:28:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:03 DEBUG --- http response --- 17:28:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:28:03 DEBUG ----- output ----- 17:28:03 DEBUG { 17:28:03 DEBUG "status": "success", 17:28:03 DEBUG "data": { 17:28:03 DEBUG "resultType": "vector", 17:28:03 DEBUG "result": [ 17:28:03 DEBUG { 17:28:03 DEBUG "metric": { 17:28:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:28:03 DEBUG }, 17:28:03 DEBUG "value": [ 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG [...] 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:28:03 DEBUG }, 17:28:03 DEBUG "value": [ 17:28:03 DEBUG 1713544083, 17:28:03 DEBUG "0" 17:28:03 DEBUG ] 17:28:03 DEBUG } 17:28:03 DEBUG ] 17:28:03 DEBUG } 17:28:03 DEBUG } 17:28:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544083" 17:28:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544083" 17:28:03 INFO [http_cmd]: http status code OK 17:28:03 DEBUG --- status code --- 17:28:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:03 DEBUG --- http response --- 17:28:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:28:03 DEBUG ----- output ----- 17:28:03 DEBUG { 17:28:03 DEBUG "status": "success", 17:28:03 DEBUG "data": { 17:28:03 DEBUG "resultType": "vector", 17:28:03 DEBUG "result": [ 17:28:03 DEBUG { 17:28:03 DEBUG "metric": { 17:28:03 DEBUG "pod": "ds-idrepo-0" 17:28:03 DEBUG }, 17:28:03 DEBUG "value": [ 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG [...] 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG "pod": "ds-cts-2" 17:28:03 DEBUG }, 17:28:03 DEBUG "value": [ 17:28:03 DEBUG 1713544083, 17:28:03 DEBUG "0" 17:28:03 DEBUG ] 17:28:03 DEBUG } 17:28:03 DEBUG ] 17:28:03 DEBUG } 17:28:03 DEBUG } 17:28:03 INFO [http_cmd]: http status code OK 17:28:03 DEBUG --- status code --- 17:28:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:03 DEBUG --- http response --- 17:28:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:28:03 DEBUG ----- output ----- 17:28:03 DEBUG { 17:28:03 DEBUG "status": "success", 17:28:03 DEBUG "data": { 17:28:03 DEBUG "resultType": "vector", 17:28:03 DEBUG "result": [ 17:28:03 DEBUG { 17:28:03 DEBUG "metric": { 17:28:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:28:03 DEBUG }, 17:28:03 DEBUG "value": [ 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG [...] 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:28:03 DEBUG }, 17:28:03 DEBUG "value": [ 17:28:03 DEBUG 1713544083, 17:28:03 DEBUG "2334.733333333333" 17:28:03 DEBUG ] 17:28:03 DEBUG } 17:28:03 DEBUG ] 17:28:03 DEBUG } 17:28:03 DEBUG } 17:28:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544083" 17:28:03 INFO [http_cmd]: http status code OK 17:28:03 DEBUG --- status code --- 17:28:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:03 DEBUG --- http response --- 17:28:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:28:03 DEBUG ----- output ----- 17:28:03 DEBUG { 17:28:03 DEBUG "status": "success", 17:28:03 DEBUG "data": { 17:28:03 DEBUG "resultType": "vector", 17:28:03 DEBUG "result": [ 17:28:03 DEBUG { 17:28:03 DEBUG "metric": { 17:28:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:28:03 DEBUG }, 17:28:03 DEBUG "value": [ 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG [...] 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:28:03 DEBUG }, 17:28:03 DEBUG "value": [ 17:28:03 DEBUG 1713544083, 17:28:03 DEBUG "0" 17:28:03 DEBUG ] 17:28:03 DEBUG } 17:28:03 DEBUG ] 17:28:03 DEBUG } 17:28:03 DEBUG } 17:28:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544083" 17:28:03 INFO [http_cmd]: http status code OK 17:28:03 DEBUG --- status code --- 17:28:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:03 DEBUG --- http response --- 17:28:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:28:03 DEBUG ----- output ----- 17:28:03 DEBUG { 17:28:03 DEBUG "status": "success", 17:28:03 DEBUG "data": { 17:28:03 DEBUG "resultType": "vector", 17:28:03 DEBUG "result": [ 17:28:03 DEBUG { 17:28:03 DEBUG "metric": { 17:28:03 DEBUG "pod": "ds-idrepo-0" 17:28:03 DEBUG }, 17:28:03 DEBUG "value": [ 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG [...] 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG "pod": "idm-b85f9d978-txpwv" 17:28:03 DEBUG }, 17:28:03 DEBUG "value": [ 17:28:03 DEBUG 1713544083, 17:28:03 DEBUG "0" 17:28:03 DEBUG ] 17:28:03 DEBUG } 17:28:03 DEBUG ] 17:28:03 DEBUG } 17:28:03 DEBUG } 17:28:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544083" 17:28:03 INFO [http_cmd]: http status code OK 17:28:03 DEBUG --- status code --- 17:28:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:03 DEBUG --- http response --- 17:28:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:28:03 DEBUG ----- output ----- 17:28:03 DEBUG { 17:28:03 DEBUG "status": "success", 17:28:03 DEBUG "data": { 17:28:03 DEBUG "resultType": "vector", 17:28:03 DEBUG "result": [ 17:28:03 DEBUG { 17:28:03 DEBUG "metric": { 17:28:03 DEBUG "instance": "10.142.0.107:9100" 17:28:03 DEBUG }, 17:28:03 DEBUG "value": [ 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG [...] 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG 17:28:03 DEBUG "instance": "10.142.0.43:9100" 17:28:03 DEBUG }, 17:28:03 DEBUG "value": [ 17:28:03 DEBUG 1713544083, 17:28:03 DEBUG "0.05" 17:28:03 DEBUG ] 17:28:03 DEBUG } 17:28:03 DEBUG ] 17:28:03 DEBUG } 17:28:03 DEBUG } 17:28:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544084" 17:28:04 INFO [http_cmd]: http status code OK 17:28:04 DEBUG --- status code --- 17:28:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:04 DEBUG --- http response --- 17:28:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:28:04 DEBUG ----- output ----- 17:28:04 DEBUG { 17:28:04 DEBUG "status": "success", 17:28:04 DEBUG "data": { 17:28:04 DEBUG "resultType": "vector", 17:28:04 DEBUG "result": [ 17:28:04 DEBUG { 17:28:04 DEBUG "metric": { 17:28:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:28:04 DEBUG }, 17:28:04 DEBUG "value": [ 17:28:04 DEBUG 17:28:04 DEBUG 17:28:04 DEBUG 17:28:04 DEBUG [...] 17:28:04 DEBUG 17:28:04 DEBUG 17:28:04 DEBUG 17:28:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:28:04 DEBUG }, 17:28:04 DEBUG "value": [ 17:28:04 DEBUG 1713544084, 17:28:04 DEBUG "3886146" 17:28:04 DEBUG ] 17:28:04 DEBUG } 17:28:04 DEBUG ] 17:28:04 DEBUG } 17:28:04 DEBUG } 17:28:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544084" 17:28:04 INFO [http_cmd]: http status code OK 17:28:04 DEBUG --- status code --- 17:28:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:04 DEBUG --- http response --- 17:28:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:28:04 DEBUG ----- output ----- 17:28:04 DEBUG { 17:28:04 DEBUG "status": "success", 17:28:04 DEBUG "data": { 17:28:04 DEBUG "resultType": "vector", 17:28:04 DEBUG "result": [ 17:28:04 DEBUG { 17:28:04 DEBUG "metric": { 17:28:04 DEBUG "instance": "10.142.0.107:9100" 17:28:04 DEBUG }, 17:28:04 DEBUG "value": [ 17:28:04 DEBUG 17:28:04 DEBUG 17:28:04 DEBUG 17:28:04 DEBUG [...] 17:28:04 DEBUG 17:28:04 DEBUG 17:28:04 DEBUG 17:28:04 DEBUG "instance": "10.142.0.43:9100" 17:28:04 DEBUG }, 17:28:04 DEBUG "value": [ 17:28:04 DEBUG 1713544084, 17:28:04 DEBUG "1702918814720" 17:28:04 DEBUG ] 17:28:04 DEBUG } 17:28:04 DEBUG ] 17:28:04 DEBUG } 17:28:04 DEBUG } 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544085" 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG { 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG "data": { 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "result": [ 17:28:05 DEBUG { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "pod": "ds-idrepo-0" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG "pod": "ds-idrepo-2" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG "0" 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544085" 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544085" 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544085" 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544085" 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG { 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG "data": { 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "result": [ 17:28:05 DEBUG { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "pod": "ds-idrepo-0" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG "pod": "ds-idrepo-2" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG "0" 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG { 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG "data": { 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "result": [ 17:28:05 DEBUG { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "pod": "ds-cts-0" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG "pod": "ds-cts-2" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG "0" 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG { 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG "data": { 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "result": [ 17:28:05 DEBUG { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG "999215104" 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG { 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG "data": { 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "result": [ 17:28:05 DEBUG { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "pod": "ds-cts-0" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG "pod": "ds-cts-2" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG "0" 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544085" 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544085" 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG { 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG "data": { 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "result": [ 17:28:05 DEBUG { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "instance": "10.142.0.107:9100" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG "instance": "10.142.0.43:9100" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG "56590754.06666666" 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG { 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG "data": { 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "result": [ 17:28:05 DEBUG { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "pod": "ds-idrepo-0" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG "pod": "ds-idrepo-2" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG "4329.9301749798315" 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544085" 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG { 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG "data": { 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "result": [ 17:28:05 DEBUG { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "instance": "10.142.0.107:9100" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG "instance": "10.142.0.43:9100" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG "56583726.56666666" 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544085" 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544085" 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544085" 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544085" 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544085" 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544085" 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544085" 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544085" 17:28:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544085" 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG --- status code --- 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG --- status code --- 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG --- http response --- 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG --- status code --- 17:28:05 INFO [http_cmd]: http status code OK 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG --- status code --- 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG { 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:28:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:28:05 DEBUG { 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG --- http response --- 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:28:05 DEBUG "data": { 17:28:05 DEBUG { 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG { 17:28:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:28:05 DEBUG "data": { 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG { 17:28:05 DEBUG { 17:28:05 DEBUG { 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG ----- output ----- 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG { 17:28:05 DEBUG "result": [ 17:28:05 DEBUG "data": { 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG "data": { 17:28:05 DEBUG { 17:28:05 DEBUG "result": [ 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG { 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "data": { 17:28:05 DEBUG "data": { 17:28:05 DEBUG "data": { 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "status": "success", 17:28:05 DEBUG { 17:28:05 DEBUG "data": { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "result": [ 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "result": [ 17:28:05 DEBUG "data": { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:28:05 DEBUG { 17:28:05 DEBUG "result": [ 17:28:05 DEBUG "result": [ 17:28:05 DEBUG "result": [ 17:28:05 DEBUG { 17:28:05 DEBUG "resultType": "vector", 17:28:05 DEBUG "pod": "ds-idrepo-0" 17:28:05 DEBUG "result": [ 17:28:05 DEBUG }, 17:28:05 DEBUG "metric": { 17:28:05 DEBUG { 17:28:05 DEBUG { 17:28:05 DEBUG { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "result": [ 17:28:05 DEBUG }, 17:28:05 DEBUG { 17:28:05 DEBUG "value": [ 17:28:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "metric": { 17:28:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:28:05 DEBUG { 17:28:05 DEBUG "value": [ 17:28:05 DEBUG "metric": { 17:28:05 DEBUG 17:28:05 DEBUG }, 17:28:05 DEBUG "pod": "ds-cts-0" 17:28:05 DEBUG "instance": "10.142.0.107:9100" 17:28:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:28:05 DEBUG }, 17:28:05 DEBUG "metric": { 17:28:05 DEBUG 17:28:05 DEBUG "pod": "ds-idrepo-0" 17:28:05 DEBUG 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG }, 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG "pod": "ds-idrepo-0" 17:28:05 DEBUG 17:28:05 DEBUG }, 17:28:05 DEBUG 17:28:05 DEBUG }, 17:28:05 DEBUG 17:28:05 DEBUG "value": [ 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG }, 17:28:05 DEBUG 17:28:05 DEBUG "value": [ 17:28:05 DEBUG [...] 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG "value": [ 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:28:05 DEBUG [...] 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG [...] 17:28:05 DEBUG "pod": "ds-idrepo-2" 17:28:05 DEBUG 17:28:05 DEBUG }, 17:28:05 DEBUG 17:28:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:28:05 DEBUG }, 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG }, 17:28:05 DEBUG 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG 17:28:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:28:05 DEBUG 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG 17:28:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:28:05 DEBUG }, 17:28:05 DEBUG 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG "pod": "ds-idrepo-2" 17:28:05 DEBUG "2334.733333333333" 17:28:05 DEBUG "pod": "ds-cts-2" 17:28:05 DEBUG "7.88072072452701" 17:28:05 DEBUG "instance": "10.142.0.43:9100" 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG "pod": "ds-idrepo-2" 17:28:05 DEBUG "5.476847692436467" 17:28:05 DEBUG }, 17:28:05 DEBUG ] 17:28:05 DEBUG }, 17:28:05 DEBUG ] 17:28:05 DEBUG }, 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG }, 17:28:05 DEBUG ] 17:28:05 DEBUG "value": [ 17:28:05 DEBUG } 17:28:05 DEBUG "value": [ 17:28:05 DEBUG } 17:28:05 DEBUG "value": [ 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG "0.023" 17:28:05 DEBUG "value": [ 17:28:05 DEBUG } 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG ] 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG ] 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG "2334.5" 17:28:05 DEBUG ] 17:28:05 DEBUG 1713544085, 17:28:05 DEBUG ] 17:28:05 DEBUG "0" 17:28:05 DEBUG } 17:28:05 DEBUG "5.3" 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG "0" 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG "0.07853333333335873" 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 DEBUG ] 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 DEBUG } 17:28:05 INFO 17:28:05 INFO [loop_until]: kubectl --namespace=xlou top pods 17:28:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:28:05 INFO 17:28:05 INFO [loop_until]: kubectl --namespace=xlou top node 17:28:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:28:05 INFO [loop_until]: OK (rc = 0) 17:28:05 DEBUG --- stdout --- 17:28:05 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 11556m 8750Mi am-6fd554f8f7-btd4s 11787m 8809Mi am-6fd554f8f7-wxcsj 12466m 8803Mi ds-cts-0 9457m 12441Mi ds-cts-1 9486m 12379Mi ds-cts-2 9525m 12425Mi ds-idrepo-0 5651m 3421Mi ds-idrepo-1 5415m 3387Mi ds-idrepo-2 5516m 3434Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7853m 863Mi 17:28:05 DEBUG --- stderr --- 17:28:05 DEBUG 17:28:05 INFO [loop_until]: OK (rc = 0) 17:28:05 DEBUG --- stdout --- 17:28:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6184Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5410m 18% 6993Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 45m 0% 2698Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11642m 38% 12028Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9452m 31% 15464Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5221Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11606m 38% 11688Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12299m 41% 2970Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13545m 45% 11019Mi 9% gke-xlou-cdm-ds-0f601751-j488 6226m 20% 6660Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9932m 33% 17177Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6195m 20% 7208Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9803m 32% 14901Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7779m 48% 4632Mi 7% 17:28:05 DEBUG --- stderr --- 17:28:05 DEBUG 17:29:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544143" 17:29:03 ERROR [http_cmd]: ERROR 17:29:03 ERROR --- status code --- 17:29:03 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:29:03 ERROR --- http response --- 17:29:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:29:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:29:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544143" 17:29:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544143" 17:29:03 INFO [http_cmd]: http status code OK 17:29:03 DEBUG --- status code --- 17:29:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:03 DEBUG --- http response --- 17:29:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:29:03 INFO [http_cmd]: http status code OK 17:29:03 DEBUG --- status code --- 17:29:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:03 DEBUG --- http response --- 17:29:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:29:03 DEBUG ----- output ----- 17:29:03 DEBUG { 17:29:03 DEBUG "status": "success", 17:29:03 DEBUG "data": { 17:29:03 DEBUG "resultType": "vector", 17:29:03 DEBUG "result": [ 17:29:03 DEBUG { 17:29:03 DEBUG "metric": { 17:29:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:29:03 DEBUG }, 17:29:03 DEBUG "value": [ 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG [...] 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:29:03 DEBUG }, 17:29:03 DEBUG "value": [ 17:29:03 DEBUG 1713544143, 17:29:03 DEBUG "0" 17:29:03 DEBUG ] 17:29:03 DEBUG } 17:29:03 DEBUG ] 17:29:03 DEBUG } 17:29:03 DEBUG } 17:29:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544143" 17:29:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544143" 17:29:03 INFO [http_cmd]: http status code OK 17:29:03 DEBUG --- status code --- 17:29:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:03 DEBUG --- http response --- 17:29:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:29:03 DEBUG ----- output ----- 17:29:03 DEBUG { 17:29:03 DEBUG "status": "success", 17:29:03 DEBUG "data": { 17:29:03 DEBUG "resultType": "vector", 17:29:03 DEBUG "result": [ 17:29:03 DEBUG { 17:29:03 DEBUG "metric": { 17:29:03 DEBUG "pod": "ds-idrepo-0" 17:29:03 DEBUG }, 17:29:03 DEBUG "value": [ 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG [...] 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG "pod": "ds-cts-2" 17:29:03 DEBUG }, 17:29:03 DEBUG "value": [ 17:29:03 DEBUG 1713544143, 17:29:03 DEBUG "0" 17:29:03 DEBUG ] 17:29:03 DEBUG } 17:29:03 DEBUG ] 17:29:03 DEBUG } 17:29:03 DEBUG } 17:29:03 INFO [http_cmd]: http status code OK 17:29:03 DEBUG --- status code --- 17:29:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:03 DEBUG --- http response --- 17:29:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:29:03 DEBUG ----- output ----- 17:29:03 DEBUG { 17:29:03 DEBUG "status": "success", 17:29:03 DEBUG "data": { 17:29:03 DEBUG "resultType": "vector", 17:29:03 DEBUG "result": [ 17:29:03 DEBUG { 17:29:03 DEBUG "metric": { 17:29:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:29:03 DEBUG }, 17:29:03 DEBUG "value": [ 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG [...] 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:29:03 DEBUG }, 17:29:03 DEBUG "value": [ 17:29:03 DEBUG 1713544143, 17:29:03 DEBUG "2338.5333333333333" 17:29:03 DEBUG ] 17:29:03 DEBUG } 17:29:03 DEBUG ] 17:29:03 DEBUG } 17:29:03 DEBUG } 17:29:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544143" 17:29:03 INFO [http_cmd]: http status code OK 17:29:03 DEBUG --- status code --- 17:29:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:03 DEBUG --- http response --- 17:29:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:29:03 DEBUG ----- output ----- 17:29:03 DEBUG { 17:29:03 DEBUG "status": "success", 17:29:03 DEBUG "data": { 17:29:03 DEBUG "resultType": "vector", 17:29:03 DEBUG "result": [ 17:29:03 DEBUG { 17:29:03 DEBUG "metric": { 17:29:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:29:03 DEBUG }, 17:29:03 DEBUG "value": [ 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG [...] 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:29:03 DEBUG }, 17:29:03 DEBUG "value": [ 17:29:03 DEBUG 1713544143, 17:29:03 DEBUG "0" 17:29:03 DEBUG ] 17:29:03 DEBUG } 17:29:03 DEBUG ] 17:29:03 DEBUG } 17:29:03 DEBUG } 17:29:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544143" 17:29:03 INFO [http_cmd]: http status code OK 17:29:03 DEBUG --- status code --- 17:29:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:03 DEBUG --- http response --- 17:29:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:29:03 DEBUG ----- output ----- 17:29:03 DEBUG { 17:29:03 DEBUG "status": "success", 17:29:03 DEBUG "data": { 17:29:03 DEBUG "resultType": "vector", 17:29:03 DEBUG "result": [ 17:29:03 DEBUG { 17:29:03 DEBUG "metric": { 17:29:03 DEBUG "pod": "ds-idrepo-0" 17:29:03 DEBUG }, 17:29:03 DEBUG "value": [ 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG [...] 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:29:03 DEBUG }, 17:29:03 DEBUG "value": [ 17:29:03 DEBUG 1713544143, 17:29:03 DEBUG "0" 17:29:03 DEBUG ] 17:29:03 DEBUG } 17:29:03 DEBUG ] 17:29:03 DEBUG } 17:29:03 DEBUG } 17:29:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544143" 17:29:03 INFO [http_cmd]: http status code OK 17:29:03 DEBUG --- status code --- 17:29:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:03 DEBUG --- http response --- 17:29:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:29:03 DEBUG ----- output ----- 17:29:03 DEBUG { 17:29:03 DEBUG "status": "success", 17:29:03 DEBUG "data": { 17:29:03 DEBUG "resultType": "vector", 17:29:03 DEBUG "result": [ 17:29:03 DEBUG { 17:29:03 DEBUG "metric": { 17:29:03 DEBUG "instance": "10.142.0.107:9100" 17:29:03 DEBUG }, 17:29:03 DEBUG "value": [ 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG [...] 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG 17:29:03 DEBUG "instance": "10.142.0.43:9100" 17:29:03 DEBUG }, 17:29:03 DEBUG "value": [ 17:29:03 DEBUG 1713544143, 17:29:03 DEBUG "0.04266666666666196" 17:29:03 DEBUG ] 17:29:03 DEBUG } 17:29:03 DEBUG ] 17:29:03 DEBUG } 17:29:03 DEBUG } 17:29:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544144" 17:29:04 INFO [http_cmd]: http status code OK 17:29:04 DEBUG --- status code --- 17:29:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:04 DEBUG --- http response --- 17:29:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:29:04 DEBUG ----- output ----- 17:29:04 DEBUG { 17:29:04 DEBUG "status": "success", 17:29:04 DEBUG "data": { 17:29:04 DEBUG "resultType": "vector", 17:29:04 DEBUG "result": [ 17:29:04 DEBUG { 17:29:04 DEBUG "metric": { 17:29:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:29:04 DEBUG }, 17:29:04 DEBUG "value": [ 17:29:04 DEBUG 17:29:04 DEBUG 17:29:04 DEBUG 17:29:04 DEBUG [...] 17:29:04 DEBUG 17:29:04 DEBUG 17:29:04 DEBUG 17:29:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:29:04 DEBUG }, 17:29:04 DEBUG "value": [ 17:29:04 DEBUG 1713544144, 17:29:04 DEBUG "4026181.5" 17:29:04 DEBUG ] 17:29:04 DEBUG } 17:29:04 DEBUG ] 17:29:04 DEBUG } 17:29:04 DEBUG } 17:29:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544144" 17:29:04 INFO [http_cmd]: http status code OK 17:29:04 DEBUG --- status code --- 17:29:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:04 DEBUG --- http response --- 17:29:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:29:04 DEBUG ----- output ----- 17:29:04 DEBUG { 17:29:04 DEBUG "status": "success", 17:29:04 DEBUG "data": { 17:29:04 DEBUG "resultType": "vector", 17:29:04 DEBUG "result": [ 17:29:04 DEBUG { 17:29:04 DEBUG "metric": { 17:29:04 DEBUG "instance": "10.142.0.107:9100" 17:29:04 DEBUG }, 17:29:04 DEBUG "value": [ 17:29:04 DEBUG 17:29:04 DEBUG 17:29:04 DEBUG 17:29:04 DEBUG [...] 17:29:04 DEBUG 17:29:04 DEBUG 17:29:04 DEBUG 17:29:04 DEBUG "instance": "10.142.0.43:9100" 17:29:04 DEBUG }, 17:29:04 DEBUG "value": [ 17:29:04 DEBUG 1713544144, 17:29:04 DEBUG "1705767269376" 17:29:04 DEBUG ] 17:29:04 DEBUG } 17:29:04 DEBUG ] 17:29:04 DEBUG } 17:29:04 DEBUG } 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544145" 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG { 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "data": { 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "result": [ 17:29:05 DEBUG { 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG "1002471424" 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544145" 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG { 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "data": { 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "result": [ 17:29:05 DEBUG { 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "pod": "ds-idrepo-0" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "pod": "ds-idrepo-2" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG "0" 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544145" 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544145" 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544145" 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG { 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "data": { 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "result": [ 17:29:05 DEBUG { 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "pod": "ds-idrepo-0" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "pod": "ds-idrepo-2" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG "0" 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG { 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "data": { 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "result": [ 17:29:05 DEBUG { 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "pod": "ds-cts-0" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "pod": "ds-cts-2" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG "0" 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG { 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "data": { 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "result": [ 17:29:05 DEBUG { 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "pod": "ds-cts-0" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "pod": "ds-cts-2" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG "0" 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544145" 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544145" 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG { 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "data": { 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "result": [ 17:29:05 DEBUG { 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "instance": "10.142.0.107:9100" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "instance": "10.142.0.43:9100" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG "56706820.8" 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG { 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "data": { 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "result": [ 17:29:05 DEBUG { 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "pod": "ds-idrepo-0" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG "0" 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544145" 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG { 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "data": { 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "result": [ 17:29:05 DEBUG { 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "instance": "10.142.0.107:9100" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "instance": "10.142.0.43:9100" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG "56699754.3" 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544145" 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544145" 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544145" 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544145" 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544145" 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544145" 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544145" 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544145" 17:29:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544145" 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG --- http response --- 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG { 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "data": { 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "result": [ 17:29:05 DEBUG { 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:29:05 DEBUG }, 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG "value": [ 17:29:05 DEBUG --- status code --- 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG { 17:29:05 DEBUG 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG 17:29:05 INFO [http_cmd]: http status code OK 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG "data": { 17:29:05 DEBUG 17:29:05 DEBUG --- status code --- 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG [...] 17:29:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG "result": [ 17:29:05 DEBUG 17:29:05 DEBUG --- http response --- 17:29:05 DEBUG { 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:29:05 DEBUG { 17:29:05 DEBUG { 17:29:05 DEBUG 17:29:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 17:29:05 DEBUG { 17:29:05 DEBUG "data": { 17:29:05 DEBUG { 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "metric": { 17:29:05 DEBUG 17:29:05 DEBUG ----- output ----- 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG { 17:29:05 DEBUG { 17:29:05 DEBUG "data": { 17:29:05 DEBUG "pod": "ds-idrepo-0" 17:29:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:29:05 DEBUG { 17:29:05 DEBUG "data": { 17:29:05 DEBUG "result": [ 17:29:05 DEBUG "data": { 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG }, 17:29:05 DEBUG }, 17:29:05 DEBUG "status": "success", 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG { 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "data": { 17:29:05 DEBUG "data": { 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "result": [ 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG "result": [ 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "result": [ 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "result": [ 17:29:05 DEBUG { 17:29:05 DEBUG "value": [ 17:29:05 DEBUG "2338.0333333333333" 17:29:05 DEBUG { 17:29:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:29:05 DEBUG "data": { 17:29:05 DEBUG { 17:29:05 DEBUG "result": [ 17:29:05 DEBUG { 17:29:05 DEBUG "metric": { 17:29:05 DEBUG 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG }, 17:29:05 DEBUG "resultType": "vector", 17:29:05 DEBUG "metric": { 17:29:05 DEBUG { 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "instance": "10.142.0.107:9100" 17:29:05 DEBUG 17:29:05 DEBUG "metric": { 17:29:05 DEBUG ] 17:29:05 DEBUG "value": [ 17:29:05 DEBUG "result": [ 17:29:05 DEBUG "pod": "ds-idrepo-0" 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:29:05 DEBUG }, 17:29:05 DEBUG 17:29:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:29:05 DEBUG } 17:29:05 DEBUG 17:29:05 DEBUG { 17:29:05 DEBUG }, 17:29:05 DEBUG "pod": "ds-cts-0" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG [...] 17:29:05 DEBUG }, 17:29:05 DEBUG } 17:29:05 DEBUG 17:29:05 DEBUG "metric": { 17:29:05 DEBUG "value": [ 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 17:29:05 DEBUG "pod": "ds-idrepo-0" 17:29:05 DEBUG 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG }, 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "value": [ 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG [...] 17:29:05 DEBUG 17:29:05 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG }, 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "value": [ 17:29:05 DEBUG "instance": "10.142.0.43:9100" 17:29:05 DEBUG }, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG }, 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG 17:29:05 DEBUG [...] 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG "0.07393333333340499" 17:29:05 DEBUG "value": [ 17:29:05 DEBUG "pod": "ds-cts-2" 17:29:05 DEBUG 17:29:05 DEBUG "0.000037913632298448545" 17:29:05 DEBUG 17:29:05 DEBUG 17:29:05 DEBUG "pod": "ds-idrepo-2" 17:29:05 DEBUG ] 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG }, 17:29:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:29:05 DEBUG ] 17:29:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:29:05 DEBUG 17:29:05 DEBUG }, 17:29:05 DEBUG } 17:29:05 DEBUG "2338.1" 17:29:05 DEBUG "value": [ 17:29:05 DEBUG }, 17:29:05 DEBUG } 17:29:05 DEBUG }, 17:29:05 DEBUG 17:29:05 DEBUG "value": [ 17:29:05 DEBUG ] 17:29:05 DEBUG ] 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG "value": [ 17:29:05 DEBUG ] 17:29:05 DEBUG "value": [ 17:29:05 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 DEBUG "5.466666666666667" 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG } 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG }, 17:29:05 DEBUG "0" 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG ] 17:29:05 DEBUG "7.884352112762853" 17:29:05 DEBUG } 17:29:05 DEBUG "0.023" 17:29:05 DEBUG "value": [ 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG ] 17:29:05 DEBUG 1713544145, 17:29:05 DEBUG "0" 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 DEBUG ] 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 DEBUG } 17:29:05 INFO 17:29:05 INFO [loop_until]: kubectl --namespace=xlou top pods 17:29:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:29:05 INFO 17:29:05 INFO [loop_until]: kubectl --namespace=xlou top node 17:29:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:29:05 INFO [loop_until]: OK (rc = 0) 17:29:05 DEBUG --- stdout --- 17:29:05 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 12023m 8752Mi am-6fd554f8f7-btd4s 11919m 8810Mi am-6fd554f8f7-wxcsj 12697m 8805Mi ds-cts-0 9596m 12441Mi ds-cts-1 9876m 12380Mi ds-cts-2 9960m 12427Mi ds-idrepo-0 5721m 3422Mi ds-idrepo-1 5422m 3387Mi ds-idrepo-2 5553m 3434Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8042m 865Mi 17:29:05 DEBUG --- stderr --- 17:29:05 DEBUG 17:29:05 INFO [loop_until]: OK (rc = 0) 17:29:05 DEBUG --- stdout --- 17:29:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6178Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5441m 18% 6993Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 47m 0% 2697Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 11362m 38% 12044Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9732m 32% 15475Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 139m 0% 5212Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11945m 40% 11694Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12370m 41% 2970Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 13568m 45% 11017Mi 9% gke-xlou-cdm-ds-0f601751-j488 6252m 20% 6652Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9882m 33% 17182Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6207m 20% 7209Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 10125m 33% 14897Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8047m 50% 4630Mi 7% 17:29:05 DEBUG --- stderr --- 17:29:05 DEBUG 17:30:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544203" 17:30:03 ERROR [http_cmd]: ERROR 17:30:03 ERROR --- status code --- 17:30:03 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:30:03 ERROR --- http response --- 17:30:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:30:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:30:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544203" 17:30:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544203" 17:30:03 INFO [http_cmd]: http status code OK 17:30:03 DEBUG --- status code --- 17:30:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:03 DEBUG --- http response --- 17:30:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:30:03 INFO [http_cmd]: http status code OK 17:30:03 DEBUG --- status code --- 17:30:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:03 DEBUG --- http response --- 17:30:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:30:03 DEBUG ----- output ----- 17:30:03 DEBUG { 17:30:03 DEBUG "status": "success", 17:30:03 DEBUG "data": { 17:30:03 DEBUG "resultType": "vector", 17:30:03 DEBUG "result": [ 17:30:03 DEBUG { 17:30:03 DEBUG "metric": { 17:30:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:30:03 DEBUG }, 17:30:03 DEBUG "value": [ 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG [...] 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:30:03 DEBUG }, 17:30:03 DEBUG "value": [ 17:30:03 DEBUG 1713544203, 17:30:03 DEBUG "289.2926733333333" 17:30:03 DEBUG ] 17:30:03 DEBUG } 17:30:03 DEBUG ] 17:30:03 DEBUG } 17:30:03 DEBUG } 17:30:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544203" 17:30:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544203" 17:30:03 INFO [http_cmd]: http status code OK 17:30:03 DEBUG --- status code --- 17:30:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:03 DEBUG --- http response --- 17:30:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:30:03 DEBUG ----- output ----- 17:30:03 DEBUG { 17:30:03 DEBUG "status": "success", 17:30:03 DEBUG "data": { 17:30:03 DEBUG "resultType": "vector", 17:30:03 DEBUG "result": [ 17:30:03 DEBUG { 17:30:03 DEBUG "metric": { 17:30:03 DEBUG "pod": "ds-idrepo-0" 17:30:03 DEBUG }, 17:30:03 DEBUG "value": [ 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG [...] 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG "pod": "ds-cts-2" 17:30:03 DEBUG }, 17:30:03 DEBUG "value": [ 17:30:03 DEBUG 1713544203, 17:30:03 DEBUG "0" 17:30:03 DEBUG ] 17:30:03 DEBUG } 17:30:03 DEBUG ] 17:30:03 DEBUG } 17:30:03 DEBUG } 17:30:03 INFO [http_cmd]: http status code OK 17:30:03 DEBUG --- status code --- 17:30:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:03 DEBUG --- http response --- 17:30:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:30:03 DEBUG ----- output ----- 17:30:03 DEBUG { 17:30:03 DEBUG "status": "success", 17:30:03 DEBUG "data": { 17:30:03 DEBUG "resultType": "vector", 17:30:03 DEBUG "result": [ 17:30:03 DEBUG { 17:30:03 DEBUG "metric": { 17:30:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:30:03 DEBUG }, 17:30:03 DEBUG "value": [ 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG [...] 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:30:03 DEBUG }, 17:30:03 DEBUG "value": [ 17:30:03 DEBUG 1713544203, 17:30:03 DEBUG "2331.6666666666665" 17:30:03 DEBUG ] 17:30:03 DEBUG } 17:30:03 DEBUG ] 17:30:03 DEBUG } 17:30:03 DEBUG } 17:30:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544203" 17:30:03 INFO [http_cmd]: http status code OK 17:30:03 DEBUG --- status code --- 17:30:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:03 DEBUG --- http response --- 17:30:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:30:03 DEBUG ----- output ----- 17:30:03 DEBUG { 17:30:03 DEBUG "status": "success", 17:30:03 DEBUG "data": { 17:30:03 DEBUG "resultType": "vector", 17:30:03 DEBUG "result": [ 17:30:03 DEBUG { 17:30:03 DEBUG "metric": { 17:30:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:30:03 DEBUG }, 17:30:03 DEBUG "value": [ 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG [...] 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:30:03 DEBUG }, 17:30:03 DEBUG "value": [ 17:30:03 DEBUG 1713544203, 17:30:03 DEBUG "0" 17:30:03 DEBUG ] 17:30:03 DEBUG } 17:30:03 DEBUG ] 17:30:03 DEBUG } 17:30:03 DEBUG } 17:30:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544203" 17:30:03 INFO [http_cmd]: http status code OK 17:30:03 DEBUG --- status code --- 17:30:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:03 DEBUG --- http response --- 17:30:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:30:03 DEBUG ----- output ----- 17:30:03 DEBUG { 17:30:03 DEBUG "status": "success", 17:30:03 DEBUG "data": { 17:30:03 DEBUG "resultType": "vector", 17:30:03 DEBUG "result": [ 17:30:03 DEBUG { 17:30:03 DEBUG "metric": { 17:30:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:30:03 DEBUG }, 17:30:03 DEBUG "value": [ 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG [...] 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG "pod": "idm-b85f9d978-txpwv" 17:30:03 DEBUG }, 17:30:03 DEBUG "value": [ 17:30:03 DEBUG 1713544203, 17:30:03 DEBUG "219.0763324764353" 17:30:03 DEBUG ] 17:30:03 DEBUG } 17:30:03 DEBUG ] 17:30:03 DEBUG } 17:30:03 DEBUG } 17:30:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544203" 17:30:03 INFO [http_cmd]: http status code OK 17:30:03 DEBUG --- status code --- 17:30:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:03 DEBUG --- http response --- 17:30:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:30:03 DEBUG ----- output ----- 17:30:03 DEBUG { 17:30:03 DEBUG "status": "success", 17:30:03 DEBUG "data": { 17:30:03 DEBUG "resultType": "vector", 17:30:03 DEBUG "result": [ 17:30:03 DEBUG { 17:30:03 DEBUG "metric": { 17:30:03 DEBUG "instance": "10.142.0.107:9100" 17:30:03 DEBUG }, 17:30:03 DEBUG "value": [ 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG [...] 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG 17:30:03 DEBUG "instance": "10.142.0.43:9100" 17:30:03 DEBUG }, 17:30:03 DEBUG "value": [ 17:30:03 DEBUG 1713544203, 17:30:03 DEBUG "0.12599999999999717" 17:30:03 DEBUG ] 17:30:03 DEBUG } 17:30:03 DEBUG ] 17:30:03 DEBUG } 17:30:03 DEBUG } 17:30:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544204" 17:30:04 INFO [http_cmd]: http status code OK 17:30:04 DEBUG --- status code --- 17:30:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:04 DEBUG --- http response --- 17:30:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:30:04 DEBUG ----- output ----- 17:30:04 DEBUG { 17:30:04 DEBUG "status": "success", 17:30:04 DEBUG "data": { 17:30:04 DEBUG "resultType": "vector", 17:30:04 DEBUG "result": [ 17:30:04 DEBUG { 17:30:04 DEBUG "metric": { 17:30:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:30:04 DEBUG }, 17:30:04 DEBUG "value": [ 17:30:04 DEBUG 17:30:04 DEBUG 17:30:04 DEBUG 17:30:04 DEBUG [...] 17:30:04 DEBUG 17:30:04 DEBUG 17:30:04 DEBUG 17:30:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:30:04 DEBUG }, 17:30:04 DEBUG "value": [ 17:30:04 DEBUG 1713544204, 17:30:04 DEBUG "4161415" 17:30:04 DEBUG ] 17:30:04 DEBUG } 17:30:04 DEBUG ] 17:30:04 DEBUG } 17:30:04 DEBUG } 17:30:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544204" 17:30:04 INFO [http_cmd]: http status code OK 17:30:04 DEBUG --- status code --- 17:30:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:04 DEBUG --- http response --- 17:30:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:30:04 DEBUG ----- output ----- 17:30:04 DEBUG { 17:30:04 DEBUG "status": "success", 17:30:04 DEBUG "data": { 17:30:04 DEBUG "resultType": "vector", 17:30:04 DEBUG "result": [ 17:30:04 DEBUG { 17:30:04 DEBUG "metric": { 17:30:04 DEBUG "instance": "10.142.0.107:9100" 17:30:04 DEBUG }, 17:30:04 DEBUG "value": [ 17:30:04 DEBUG 17:30:04 DEBUG 17:30:04 DEBUG 17:30:04 DEBUG [...] 17:30:04 DEBUG 17:30:04 DEBUG 17:30:04 DEBUG 17:30:04 DEBUG "instance": "10.142.0.43:9100" 17:30:04 DEBUG }, 17:30:04 DEBUG "value": [ 17:30:04 DEBUG 1713544204, 17:30:04 DEBUG "1708628909056" 17:30:04 DEBUG ] 17:30:04 DEBUG } 17:30:04 DEBUG ] 17:30:04 DEBUG } 17:30:04 DEBUG } 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544205" 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG { 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG "data": { 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG "result": [ 17:30:05 DEBUG { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG "1004494848" 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544205" 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG { 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG "data": { 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG "result": [ 17:30:05 DEBUG { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "pod": "ds-idrepo-0" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "pod": "ds-idrepo-2" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG "0" 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544205" 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544205" 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544205" 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG { 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG "data": { 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG "result": [ 17:30:05 DEBUG { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "pod": "ds-idrepo-0" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "pod": "ds-idrepo-2" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 1713544205, 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG "0" 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG ] 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG } 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG { 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG "data": { 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG { 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG "data": { 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG "result": [ 17:30:05 DEBUG { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "pod": "ds-cts-0" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "pod": "ds-cts-2" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG "0.001154228888888889" 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG } 17:30:05 DEBUG "result": [ 17:30:05 DEBUG { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "pod": "ds-cts-0" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "pod": "ds-cts-2" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG "0" 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544205" 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544205" 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG { 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG "data": { 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG "result": [ 17:30:05 DEBUG { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG "1263.2444657665685" 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG { 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG "data": { 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG "result": [ 17:30:05 DEBUG { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "instance": "10.142.0.107:9100" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "instance": "10.142.0.43:9100" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG "58611244.03333333" 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544205" 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG { 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG "data": { 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG "result": [ 17:30:05 DEBUG { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "instance": "10.142.0.107:9100" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "instance": "10.142.0.43:9100" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG "58605506.93333334" 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544205" 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544205" 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544205" 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544205" 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544205" 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544205" 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544205" 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544205" 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544205" 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG --- status code --- 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG --- status code --- 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG --- http response --- 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG { 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG { 17:30:05 DEBUG { 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:30:05 DEBUG { 17:30:05 DEBUG "status": "success", 17:30:05 INFO [http_cmd]: http status code OK 17:30:05 DEBUG { 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:30:05 DEBUG "data": { 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG { 17:30:05 DEBUG "data": { 17:30:05 DEBUG --- status code --- 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG "data": { 17:30:05 DEBUG "data": { 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG { 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:30:05 DEBUG "data": { 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG { 17:30:05 DEBUG "result": [ 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG "data": { 17:30:05 DEBUG "result": [ 17:30:05 DEBUG --- http response --- 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG "result": [ 17:30:05 DEBUG "result": [ 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG { 17:30:05 DEBUG "data": { 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG { 17:30:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:30:05 DEBUG "result": [ 17:30:05 DEBUG { 17:30:05 DEBUG { 17:30:05 DEBUG "data": { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG "result": [ 17:30:05 DEBUG "metric": { 17:30:05 DEBUG ----- output ----- 17:30:05 DEBUG { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:30:05 DEBUG "result": [ 17:30:05 DEBUG { 17:30:05 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:30:05 DEBUG { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "instance": "10.142.0.107:9100" 17:30:05 DEBUG "pod": "ds-idrepo-0" 17:30:05 DEBUG "result": [ 17:30:05 DEBUG }, 17:30:05 DEBUG { 17:30:05 DEBUG "metric": { 17:30:05 DEBUG }, 17:30:05 DEBUG "status": "success", 17:30:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:30:05 DEBUG }, 17:30:05 DEBUG }, 17:30:05 DEBUG { 17:30:05 DEBUG "value": [ 17:30:05 DEBUG "metric": { 17:30:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:30:05 DEBUG "value": [ 17:30:05 DEBUG "data": { 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG "value": [ 17:30:05 DEBUG "metric": { 17:30:05 DEBUG 17:30:05 DEBUG "pod": "ds-cts-0" 17:30:05 DEBUG }, 17:30:05 DEBUG 17:30:05 DEBUG "resultType": "vector", 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:30:05 DEBUG 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG "result": [ 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG }, 17:30:05 DEBUG 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG { 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "value": [ 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG "metric": { 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG }, 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG [...] 17:30:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "pod": "idm-b85f9d978-txpwv" 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "instance": "10.142.0.43:9100" 17:30:05 DEBUG "pod": "ds-idrepo-2" 17:30:05 DEBUG 17:30:05 DEBUG }, 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG }, 17:30:05 DEBUG 17:30:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:30:05 DEBUG }, 17:30:05 DEBUG }, 17:30:05 DEBUG 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG "pod": "ds-cts-2" 17:30:05 DEBUG }, 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG [...] 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:30:05 DEBUG "2332.4" 17:30:05 DEBUG }, 17:30:05 DEBUG "value": [ 17:30:05 DEBUG "0" 17:30:05 DEBUG 17:30:05 DEBUG 17:30:05 DEBUG "0.21159999999999096" 17:30:05 DEBUG "0" 17:30:05 DEBUG }, 17:30:05 DEBUG ] 17:30:05 DEBUG "value": [ 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG ] 17:30:05 DEBUG 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG ] 17:30:05 DEBUG ] 17:30:05 DEBUG "value": [ 17:30:05 DEBUG } 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG "0.023" 17:30:05 DEBUG } 17:30:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:30:05 DEBUG "2332.766666666667" 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG ] 17:30:05 DEBUG "279.7" 17:30:05 DEBUG ] 17:30:05 DEBUG ] 17:30:05 DEBUG }, 17:30:05 DEBUG ] 17:30:05 DEBUG ] 17:30:05 DEBUG ] 17:30:05 DEBUG "0" 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 DEBUG "value": [ 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG 1713544205, 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG "7.845334533299388" 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 DEBUG ] 17:30:05 DEBUG } 17:30:05 DEBUG } 17:30:05 INFO 17:30:05 INFO [loop_until]: kubectl --namespace=xlou top pods 17:30:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:30:05 INFO 17:30:05 INFO [loop_until]: kubectl --namespace=xlou top node 17:30:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:30:05 INFO [loop_until]: OK (rc = 0) 17:30:05 DEBUG --- stdout --- 17:30:05 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 14171m 8876Mi am-6fd554f8f7-btd4s 13415m 8906Mi am-6fd554f8f7-wxcsj 13749m 8883Mi ds-cts-0 14153m 12606Mi ds-cts-1 14775m 12559Mi ds-cts-2 15609m 12645Mi ds-idrepo-0 5803m 3422Mi ds-idrepo-1 5641m 3387Mi ds-idrepo-2 5566m 3434Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7597m 867Mi 17:30:05 DEBUG --- stderr --- 17:30:05 DEBUG 17:30:06 INFO [loop_until]: OK (rc = 0) 17:30:06 DEBUG --- stdout --- 17:30:06 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6187Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5996m 20% 6994Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 48m 0% 2678Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 13597m 45% 12139Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 14497m 48% 15611Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5214Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13690m 45% 11767Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12449m 41% 2975Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15815m 52% 11088Mi 9% gke-xlou-cdm-ds-0f601751-j488 6306m 21% 6665Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 15361m 51% 17371Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6645m 22% 7211Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 15866m 53% 15045Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8085m 50% 4636Mi 7% 17:30:06 DEBUG --- stderr --- 17:30:06 DEBUG 17:30:17 WARNING File /tmp/lodemon_data-20240419_165801/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [19/Apr/2024 17:30:17] "GET /monitoring/average?start_time=20240419_160103&stop_time=20240419_162816 HTTP/1.1" 200 - 17:31:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544263" 17:31:03 ERROR [http_cmd]: ERROR 17:31:03 ERROR --- status code --- 17:31:03 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:31:03 ERROR --- http response --- 17:31:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:31:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:31:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544263" 17:31:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544263" 17:31:03 INFO [http_cmd]: http status code OK 17:31:03 DEBUG --- status code --- 17:31:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:03 DEBUG --- http response --- 17:31:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:31:03 INFO [http_cmd]: http status code OK 17:31:03 DEBUG --- status code --- 17:31:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:03 DEBUG --- http response --- 17:31:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:31:03 DEBUG ----- output ----- 17:31:03 DEBUG { 17:31:03 DEBUG "status": "success", 17:31:03 DEBUG "data": { 17:31:03 DEBUG "resultType": "vector", 17:31:03 DEBUG "result": [ 17:31:03 DEBUG { 17:31:03 DEBUG "metric": { 17:31:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:31:03 DEBUG }, 17:31:03 DEBUG "value": [ 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG [...] 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:31:03 DEBUG }, 17:31:03 DEBUG "value": [ 17:31:03 DEBUG 1713544263, 17:31:03 DEBUG "2343.8" 17:31:03 DEBUG ] 17:31:03 DEBUG } 17:31:03 DEBUG ] 17:31:03 DEBUG } 17:31:03 DEBUG } 17:31:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544263" 17:31:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544263" 17:31:03 INFO [http_cmd]: http status code OK 17:31:03 DEBUG --- status code --- 17:31:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:03 DEBUG --- http response --- 17:31:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:31:03 DEBUG ----- output ----- 17:31:03 DEBUG { 17:31:03 DEBUG "status": "success", 17:31:03 DEBUG "data": { 17:31:03 DEBUG "resultType": "vector", 17:31:03 DEBUG "result": [ 17:31:03 DEBUG { 17:31:03 DEBUG "metric": { 17:31:03 DEBUG "pod": "ds-idrepo-0" 17:31:03 DEBUG }, 17:31:03 DEBUG "value": [ 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG [...] 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG "pod": "ds-cts-2" 17:31:03 DEBUG }, 17:31:03 DEBUG "value": [ 17:31:03 DEBUG 1713544263, 17:31:03 DEBUG "0" 17:31:03 DEBUG ] 17:31:03 DEBUG } 17:31:03 DEBUG ] 17:31:03 DEBUG } 17:31:03 DEBUG } 17:31:03 INFO [http_cmd]: http status code OK 17:31:03 DEBUG --- status code --- 17:31:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:03 DEBUG --- http response --- 17:31:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:31:03 DEBUG ----- output ----- 17:31:03 DEBUG { 17:31:03 DEBUG "status": "success", 17:31:03 DEBUG "data": { 17:31:03 DEBUG "resultType": "vector", 17:31:03 DEBUG "result": [ 17:31:03 DEBUG { 17:31:03 DEBUG "metric": { 17:31:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:31:03 DEBUG }, 17:31:03 DEBUG "value": [ 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG [...] 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:31:03 DEBUG }, 17:31:03 DEBUG "value": [ 17:31:03 DEBUG 1713544263, 17:31:03 DEBUG "4.633333333333333" 17:31:03 DEBUG ] 17:31:03 DEBUG } 17:31:03 DEBUG ] 17:31:03 DEBUG } 17:31:03 DEBUG } 17:31:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544263" 17:31:03 INFO [http_cmd]: http status code OK 17:31:03 DEBUG --- status code --- 17:31:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:03 DEBUG --- http response --- 17:31:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:31:03 DEBUG ----- output ----- 17:31:03 DEBUG { 17:31:03 DEBUG "status": "success", 17:31:03 DEBUG "data": { 17:31:03 DEBUG "resultType": "vector", 17:31:03 DEBUG "result": [ 17:31:03 DEBUG { 17:31:03 DEBUG "metric": { 17:31:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:31:03 DEBUG }, 17:31:03 DEBUG "value": [ 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG [...] 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:31:03 DEBUG }, 17:31:03 DEBUG "value": [ 17:31:03 DEBUG 1713544263, 17:31:03 DEBUG "0" 17:31:03 DEBUG ] 17:31:03 DEBUG } 17:31:03 DEBUG ] 17:31:03 DEBUG } 17:31:03 DEBUG } 17:31:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544263" 17:31:03 INFO [http_cmd]: http status code OK 17:31:03 DEBUG --- status code --- 17:31:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:03 DEBUG --- http response --- 17:31:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:31:03 DEBUG ----- output ----- 17:31:03 DEBUG { 17:31:03 DEBUG "status": "success", 17:31:03 DEBUG "data": { 17:31:03 DEBUG "resultType": "vector", 17:31:03 DEBUG "result": [ 17:31:03 DEBUG { 17:31:03 DEBUG "metric": { 17:31:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:31:03 DEBUG }, 17:31:03 DEBUG "value": [ 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG [...] 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG "pod": "ds-idrepo-1" 17:31:03 DEBUG }, 17:31:03 DEBUG "value": [ 17:31:03 DEBUG 1713544263, 17:31:03 DEBUG "155277438.89845094" 17:31:03 DEBUG ] 17:31:03 DEBUG } 17:31:03 DEBUG ] 17:31:03 DEBUG } 17:31:03 DEBUG } 17:31:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544263" 17:31:03 INFO [http_cmd]: http status code OK 17:31:03 DEBUG --- status code --- 17:31:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:03 DEBUG --- http response --- 17:31:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:31:03 DEBUG ----- output ----- 17:31:03 DEBUG { 17:31:03 DEBUG "status": "success", 17:31:03 DEBUG "data": { 17:31:03 DEBUG "resultType": "vector", 17:31:03 DEBUG "result": [ 17:31:03 DEBUG { 17:31:03 DEBUG "metric": { 17:31:03 DEBUG "instance": "10.142.0.107:9100" 17:31:03 DEBUG }, 17:31:03 DEBUG "value": [ 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG [...] 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG 17:31:03 DEBUG "instance": "10.142.0.43:9100" 17:31:03 DEBUG }, 17:31:03 DEBUG "value": [ 17:31:03 DEBUG 1713544263, 17:31:03 DEBUG "0.48700000000000243" 17:31:03 DEBUG ] 17:31:03 DEBUG } 17:31:03 DEBUG ] 17:31:03 DEBUG } 17:31:03 DEBUG } 17:31:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544264" 17:31:04 INFO [http_cmd]: http status code OK 17:31:04 DEBUG --- status code --- 17:31:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:04 DEBUG --- http response --- 17:31:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:31:04 DEBUG ----- output ----- 17:31:04 DEBUG { 17:31:04 DEBUG "status": "success", 17:31:04 DEBUG "data": { 17:31:04 DEBUG "resultType": "vector", 17:31:04 DEBUG "result": [ 17:31:04 DEBUG { 17:31:04 DEBUG "metric": { 17:31:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:31:04 DEBUG }, 17:31:04 DEBUG "value": [ 17:31:04 DEBUG 17:31:04 DEBUG 17:31:04 DEBUG 17:31:04 DEBUG [...] 17:31:04 DEBUG 17:31:04 DEBUG 17:31:04 DEBUG 17:31:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:31:04 DEBUG }, 17:31:04 DEBUG "value": [ 17:31:04 DEBUG 1713544264, 17:31:04 DEBUG "4128000.5" 17:31:04 DEBUG ] 17:31:04 DEBUG } 17:31:04 DEBUG ] 17:31:04 DEBUG } 17:31:04 DEBUG } 17:31:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544264" 17:31:04 INFO [http_cmd]: http status code OK 17:31:04 DEBUG --- status code --- 17:31:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:04 DEBUG --- http response --- 17:31:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:31:04 DEBUG ----- output ----- 17:31:04 DEBUG { 17:31:04 DEBUG "status": "success", 17:31:04 DEBUG "data": { 17:31:04 DEBUG "resultType": "vector", 17:31:04 DEBUG "result": [ 17:31:04 DEBUG { 17:31:04 DEBUG "metric": { 17:31:04 DEBUG "instance": "10.142.0.107:9100" 17:31:04 DEBUG }, 17:31:04 DEBUG "value": [ 17:31:04 DEBUG 17:31:04 DEBUG 17:31:04 DEBUG 17:31:04 DEBUG [...] 17:31:04 DEBUG 17:31:04 DEBUG 17:31:04 DEBUG 17:31:04 DEBUG "instance": "10.142.0.43:9100" 17:31:04 DEBUG }, 17:31:04 DEBUG "value": [ 17:31:04 DEBUG 1713544264, 17:31:04 DEBUG "1710484599808" 17:31:04 DEBUG ] 17:31:04 DEBUG } 17:31:04 DEBUG ] 17:31:04 DEBUG } 17:31:04 DEBUG } 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544265" 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG --- status code --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG "data": { 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "result": [ 17:31:05 DEBUG { 17:31:05 DEBUG "metric": { 17:31:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG "481677312" 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544265" 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG --- status code --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG "data": { 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "result": [ 17:31:05 DEBUG { 17:31:05 DEBUG "metric": { 17:31:05 DEBUG "pod": "ds-idrepo-0" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "pod": "ds-idrepo-2" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG "0" 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544265" 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544265" 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544265" 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG --- status code --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG "data": { 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "result": [ 17:31:05 DEBUG { 17:31:05 DEBUG "metric": { 17:31:05 DEBUG "pod": "ds-idrepo-0" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "pod": "ds-idrepo-2" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG "0" 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG --- status code --- 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- status code --- 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG "data": { 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "result": [ 17:31:05 DEBUG { 17:31:05 DEBUG "metric": { 17:31:05 DEBUG "pod": "ds-cts-0" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:31:05 DEBUG 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG 17:31:05 DEBUG { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG 17:31:05 DEBUG "data": { 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG 17:31:05 DEBUG "result": [ 17:31:05 DEBUG 17:31:05 DEBUG { 17:31:05 DEBUG "pod": "ds-cts-2" 17:31:05 DEBUG "metric": { 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG "0" 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG "pod": "ds-cts-0" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "pod": "ds-cts-2" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG "0.00009999999999999998" 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544265" 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544265" 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG --- status code --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG "data": { 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "result": [ 17:31:05 DEBUG { 17:31:05 DEBUG "metric": { 17:31:05 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "pod": "ds-cts-1" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG "12987270.891117273" 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG --- status code --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG "data": { 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "result": [ 17:31:05 DEBUG { 17:31:05 DEBUG "metric": { 17:31:05 DEBUG "instance": "10.142.0.107:9100" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "instance": "10.142.0.43:9100" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG "16298173.166666668" 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544265" 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG --- status code --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG "data": { 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "result": [ 17:31:05 DEBUG { 17:31:05 DEBUG "metric": { 17:31:05 DEBUG "instance": "10.142.0.107:9100" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "instance": "10.142.0.43:9100" 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG "16289072.6" 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544265" 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544265" 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544265" 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544265" 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544265" 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544265" 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544265" 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544265" 17:31:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544265" 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG --- status code --- 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG --- status code --- 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG --- status code --- 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG --- http response --- 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG --- status code --- 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- status code --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- status code --- 17:31:05 INFO [http_cmd]: http status code OK 17:31:05 DEBUG --- status code --- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- status code --- 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- status code --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG { 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:31:05 DEBUG --- http response --- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:31:05 DEBUG { 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG { 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG "data": { 17:31:05 DEBUG { 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG { 17:31:05 DEBUG ----- output ----- 17:31:05 DEBUG { 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG { 17:31:05 DEBUG "data": { 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "data": { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG "result": [ 17:31:05 DEBUG "data": { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG "data": { 17:31:05 DEBUG "result": [ 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "data": { 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG "data": { 17:31:05 DEBUG { 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "result": [ 17:31:05 DEBUG { 17:31:05 DEBUG "result": [ 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "data": { 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "metric": { 17:31:05 DEBUG "result": [ 17:31:05 DEBUG "status": "success", 17:31:05 DEBUG { 17:31:05 DEBUG "metric": { 17:31:05 DEBUG { 17:31:05 DEBUG "result": [ 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "result": [ 17:31:05 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:31:05 DEBUG { 17:31:05 DEBUG "data": { 17:31:05 DEBUG "metric": { 17:31:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:31:05 DEBUG "metric": { 17:31:05 DEBUG { 17:31:05 DEBUG "result": [ 17:31:05 DEBUG { 17:31:05 DEBUG }, 17:31:05 DEBUG "metric": { 17:31:05 DEBUG "resultType": "vector", 17:31:05 DEBUG "instance": "10.142.0.107:9100" 17:31:05 DEBUG }, 17:31:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:31:05 DEBUG "metric": { 17:31:05 DEBUG { 17:31:05 DEBUG "metric": { 17:31:05 DEBUG "value": [ 17:31:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:31:05 DEBUG "result": [ 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG }, 17:31:05 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:31:05 DEBUG "metric": { 17:31:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:31:05 DEBUG 17:31:05 DEBUG }, 17:31:05 DEBUG { 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 17:31:05 DEBUG "value": [ 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "pod": "ds-idrepo-0" 17:31:05 DEBUG "value": [ 17:31:05 DEBUG "metric": { 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG }, 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG }, 17:31:05 DEBUG 17:31:05 DEBUG "pod": "ds-cts-0" 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 17:31:05 DEBUG }, 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "value": [ 17:31:05 DEBUG "pod": "ds-cts-1" 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG }, 17:31:05 DEBUG 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "value": [ 17:31:05 DEBUG [...] 17:31:05 DEBUG [...] 17:31:05 DEBUG 17:31:05 DEBUG }, 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "pod": "ds-idrepo-2" 17:31:05 DEBUG "value": [ 17:31:05 DEBUG "pod": "ds-cts-1" 17:31:05 DEBUG 17:31:05 DEBUG "instance": "10.142.0.43:9100" 17:31:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:31:05 DEBUG "4.633333333333333" 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG }, 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG }, 17:31:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:31:05 DEBUG }, 17:31:05 DEBUG }, 17:31:05 DEBUG ] 17:31:05 DEBUG 17:31:05 DEBUG 17:31:05 DEBUG "value": [ 17:31:05 DEBUG "25741936.988834362" 17:31:05 DEBUG "value": [ 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG "value": [ 17:31:05 DEBUG } 17:31:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:31:05 DEBUG "pod": "ds-cts-2" 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG ] 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG }, 17:31:05 DEBUG "0" 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG "9.222528980869614" 17:31:05 DEBUG "0.030373859998362885" 17:31:05 DEBUG } 17:31:05 DEBUG }, 17:31:05 DEBUG "value": [ 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG "0.5714666666666204" 17:31:05 DEBUG "4.7" 17:31:05 DEBUG ] 17:31:05 DEBUG ] 17:31:05 DEBUG "value": [ 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG "634.6333333333333" 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG 1713544265, 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG "0.023" 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:05 DEBUG ] 17:31:05 DEBUG } 17:31:05 DEBUG } 17:31:06 INFO 17:31:06 INFO [loop_until]: kubectl --namespace=xlou top pods 17:31:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:31:06 INFO 17:31:06 INFO [loop_until]: kubectl --namespace=xlou top node 17:31:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:31:06 INFO [loop_until]: OK (rc = 0) 17:31:06 DEBUG --- stdout --- 17:31:06 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 2765m 8680Mi am-6fd554f8f7-btd4s 3513m 8876Mi am-6fd554f8f7-wxcsj 2873m 8895Mi ds-cts-0 9938m 13427Mi ds-cts-1 10039m 13395Mi ds-cts-2 11352m 12770Mi ds-idrepo-0 993m 3422Mi ds-idrepo-1 1005m 3387Mi ds-idrepo-2 881m 3433Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 2866m 794Mi 17:31:06 DEBUG --- stderr --- 17:31:06 DEBUG 17:31:06 INFO [loop_until]: OK (rc = 0) 17:31:06 DEBUG --- stdout --- 17:31:06 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6180Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 1594m 5% 6994Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 47m 0% 2685Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 2886m 9% 11989Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 10478m 35% 16491Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 140m 0% 5226Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 3268m 10% 11762Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 989m 3% 2948Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 3587m 12% 11121Mi 9% gke-xlou-cdm-ds-0f601751-j488 1746m 5% 6660Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 11015m 36% 18262Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 1572m 5% 7205Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 11139m 37% 15943Mi 14% gke-xlou-cdm-frontend-a8771548-i82n 3609m 22% 4677Mi 7% 17:31:06 DEBUG --- stderr --- 17:31:06 DEBUG 17:32:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544323" 17:32:03 ERROR [http_cmd]: ERROR 17:32:03 ERROR --- status code --- 17:32:03 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:32:03 ERROR --- http response --- 17:32:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:32:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:32:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544323" 17:32:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544323" 17:32:03 INFO [http_cmd]: http status code OK 17:32:03 DEBUG --- status code --- 17:32:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:03 DEBUG --- http response --- 17:32:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:32:03 INFO [http_cmd]: http status code OK 17:32:03 DEBUG --- status code --- 17:32:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:03 DEBUG --- http response --- 17:32:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:32:03 DEBUG ----- output ----- 17:32:03 DEBUG { 17:32:03 DEBUG "status": "success", 17:32:03 DEBUG "data": { 17:32:03 DEBUG "resultType": "vector", 17:32:03 DEBUG "result": [ 17:32:03 DEBUG { 17:32:03 DEBUG "metric": { 17:32:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:32:03 DEBUG }, 17:32:03 DEBUG "value": [ 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG [...] 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:32:03 DEBUG }, 17:32:03 DEBUG "value": [ 17:32:03 DEBUG 1713544323, 17:32:03 DEBUG "2316.9333333333334" 17:32:03 DEBUG ] 17:32:03 DEBUG } 17:32:03 DEBUG ] 17:32:03 DEBUG } 17:32:03 DEBUG } 17:32:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544323" 17:32:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544323" 17:32:03 INFO [http_cmd]: http status code OK 17:32:03 INFO [http_cmd]: http status code OK 17:32:03 DEBUG --- status code --- 17:32:03 DEBUG --- status code --- 17:32:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:03 DEBUG --- http response --- 17:32:03 DEBUG --- http response --- 17:32:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:32:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:32:03 DEBUG ----- output ----- 17:32:03 DEBUG ----- output ----- 17:32:03 DEBUG { 17:32:03 DEBUG { 17:32:03 DEBUG "status": "success", 17:32:03 DEBUG "status": "success", 17:32:03 DEBUG "data": { 17:32:03 DEBUG "data": { 17:32:03 DEBUG "resultType": "vector", 17:32:03 DEBUG "result": [ 17:32:03 DEBUG { 17:32:03 DEBUG "metric": { 17:32:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:32:03 DEBUG }, 17:32:03 DEBUG "value": [ 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG [...] 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:32:03 DEBUG }, 17:32:03 DEBUG "value": [ 17:32:03 DEBUG 1713544323, 17:32:03 DEBUG "resultType": "vector", 17:32:03 DEBUG "2479.866666666667" 17:32:03 DEBUG "result": [ 17:32:03 DEBUG ] 17:32:03 DEBUG { 17:32:03 DEBUG } 17:32:03 DEBUG "metric": { 17:32:03 DEBUG ] 17:32:03 DEBUG "pod": "ds-idrepo-0" 17:32:03 DEBUG } 17:32:03 DEBUG }, 17:32:03 DEBUG } 17:32:03 DEBUG "value": [ 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG [...] 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG "pod": "ds-cts-2" 17:32:03 DEBUG }, 17:32:03 DEBUG "value": [ 17:32:03 DEBUG 1713544323, 17:32:03 DEBUG "0" 17:32:03 DEBUG ] 17:32:03 DEBUG } 17:32:03 DEBUG ] 17:32:03 DEBUG } 17:32:03 DEBUG } 17:32:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544323" 17:32:03 INFO [http_cmd]: http status code OK 17:32:03 DEBUG --- status code --- 17:32:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:03 DEBUG --- http response --- 17:32:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:32:03 DEBUG ----- output ----- 17:32:03 DEBUG { 17:32:03 DEBUG "status": "success", 17:32:03 DEBUG "data": { 17:32:03 DEBUG "resultType": "vector", 17:32:03 DEBUG "result": [ 17:32:03 DEBUG { 17:32:03 DEBUG "metric": { 17:32:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:32:03 DEBUG }, 17:32:03 DEBUG "value": [ 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG [...] 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:32:03 DEBUG }, 17:32:03 DEBUG "value": [ 17:32:03 DEBUG 1713544323, 17:32:03 DEBUG "0" 17:32:03 DEBUG ] 17:32:03 DEBUG } 17:32:03 DEBUG ] 17:32:03 DEBUG } 17:32:03 DEBUG } 17:32:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544323" 17:32:03 INFO [http_cmd]: http status code OK 17:32:03 DEBUG --- status code --- 17:32:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:03 DEBUG --- http response --- 17:32:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:32:03 DEBUG ----- output ----- 17:32:03 DEBUG { 17:32:03 DEBUG "status": "success", 17:32:03 DEBUG "data": { 17:32:03 DEBUG "resultType": "vector", 17:32:03 DEBUG "result": [ 17:32:03 DEBUG { 17:32:03 DEBUG "metric": { 17:32:03 DEBUG "pod": "ds-idrepo-0" 17:32:03 DEBUG }, 17:32:03 DEBUG "value": [ 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG [...] 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG 17:32:03 DEBUG "pod": "ds-idrepo-2" 17:32:03 DEBUG }, 17:32:03 DEBUG "value": [ 17:32:03 DEBUG 1713544323, 17:32:03 DEBUG "0" 17:32:03 DEBUG ] 17:32:03 DEBUG } 17:32:03 DEBUG ] 17:32:03 DEBUG } 17:32:03 DEBUG } 17:32:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544324" 17:32:04 INFO [http_cmd]: http status code OK 17:32:04 DEBUG --- status code --- 17:32:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:04 DEBUG --- http response --- 17:32:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:32:04 DEBUG ----- output ----- 17:32:04 DEBUG { 17:32:04 DEBUG "status": "success", 17:32:04 DEBUG "data": { 17:32:04 DEBUG "resultType": "vector", 17:32:04 DEBUG "result": [ 17:32:04 DEBUG { 17:32:04 DEBUG "metric": { 17:32:04 DEBUG "instance": "10.142.0.107:9100" 17:32:04 DEBUG }, 17:32:04 DEBUG "value": [ 17:32:04 DEBUG 17:32:04 DEBUG 17:32:04 DEBUG 17:32:04 DEBUG [...] 17:32:04 DEBUG 17:32:04 DEBUG 17:32:04 DEBUG 17:32:04 DEBUG "instance": "10.142.0.43:9100" 17:32:04 DEBUG }, 17:32:04 DEBUG "value": [ 17:32:04 DEBUG 1713544324, 17:32:04 DEBUG "0.330666666666671" 17:32:04 DEBUG ] 17:32:04 DEBUG } 17:32:04 DEBUG ] 17:32:04 DEBUG } 17:32:04 DEBUG } 17:32:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544324" 17:32:04 INFO [http_cmd]: http status code OK 17:32:04 DEBUG --- status code --- 17:32:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:04 DEBUG --- http response --- 17:32:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:32:04 DEBUG ----- output ----- 17:32:04 DEBUG { 17:32:04 DEBUG "status": "success", 17:32:04 DEBUG "data": { 17:32:04 DEBUG "resultType": "vector", 17:32:04 DEBUG "result": [ 17:32:04 DEBUG { 17:32:04 DEBUG "metric": { 17:32:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:32:04 DEBUG }, 17:32:04 DEBUG "value": [ 17:32:04 DEBUG 17:32:04 DEBUG 17:32:04 DEBUG 17:32:04 DEBUG [...] 17:32:04 DEBUG 17:32:04 DEBUG 17:32:04 DEBUG 17:32:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:32:04 DEBUG }, 17:32:04 DEBUG "value": [ 17:32:04 DEBUG 1713544324, 17:32:04 DEBUG "4066808" 17:32:04 DEBUG ] 17:32:04 DEBUG } 17:32:04 DEBUG ] 17:32:04 DEBUG } 17:32:04 DEBUG } 17:32:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544324" 17:32:04 INFO [http_cmd]: http status code OK 17:32:04 DEBUG --- status code --- 17:32:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:04 DEBUG --- http response --- 17:32:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:32:04 DEBUG ----- output ----- 17:32:04 DEBUG { 17:32:04 DEBUG "status": "success", 17:32:04 DEBUG "data": { 17:32:04 DEBUG "resultType": "vector", 17:32:04 DEBUG "result": [ 17:32:04 DEBUG { 17:32:04 DEBUG "metric": { 17:32:04 DEBUG "instance": "10.142.0.107:9100" 17:32:04 DEBUG }, 17:32:04 DEBUG "value": [ 17:32:04 DEBUG 17:32:04 DEBUG 17:32:04 DEBUG 17:32:04 DEBUG [...] 17:32:04 DEBUG 17:32:04 DEBUG 17:32:04 DEBUG 17:32:04 DEBUG "instance": "10.142.0.43:9100" 17:32:04 DEBUG }, 17:32:04 DEBUG "value": [ 17:32:04 DEBUG 1713544324, 17:32:04 DEBUG "1712627946496" 17:32:04 DEBUG ] 17:32:04 DEBUG } 17:32:04 DEBUG ] 17:32:04 DEBUG } 17:32:04 DEBUG } 17:32:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544325" 17:32:05 INFO [http_cmd]: http status code OK 17:32:05 DEBUG --- status code --- 17:32:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:05 DEBUG --- http response --- 17:32:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:32:05 DEBUG ----- output ----- 17:32:05 DEBUG { 17:32:05 DEBUG "status": "success", 17:32:05 DEBUG "data": { 17:32:05 DEBUG "resultType": "vector", 17:32:05 DEBUG "result": [ 17:32:05 DEBUG { 17:32:05 DEBUG "metric": { 17:32:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:32:05 DEBUG }, 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG [...] 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:32:05 DEBUG }, 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 1713544325, 17:32:05 DEBUG "1247760384" 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG } 17:32:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544325" 17:32:05 INFO [http_cmd]: http status code OK 17:32:05 DEBUG --- status code --- 17:32:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:05 DEBUG --- http response --- 17:32:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:32:05 DEBUG ----- output ----- 17:32:05 DEBUG { 17:32:05 DEBUG "status": "success", 17:32:05 DEBUG "data": { 17:32:05 DEBUG "resultType": "vector", 17:32:05 DEBUG "result": [ 17:32:05 DEBUG { 17:32:05 DEBUG "metric": { 17:32:05 DEBUG "pod": "ds-idrepo-0" 17:32:05 DEBUG }, 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG [...] 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG "pod": "ds-idrepo-2" 17:32:05 DEBUG }, 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 1713544325, 17:32:05 DEBUG "0" 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG } 17:32:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544325" 17:32:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544325" 17:32:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544325" 17:32:05 INFO [http_cmd]: http status code OK 17:32:05 DEBUG --- status code --- 17:32:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:05 DEBUG --- http response --- 17:32:05 INFO [http_cmd]: http status code OK 17:32:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:32:05 DEBUG --- status code --- 17:32:05 DEBUG ----- output ----- 17:32:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:05 DEBUG { 17:32:05 DEBUG --- http response --- 17:32:05 INFO [http_cmd]: http status code OK 17:32:05 DEBUG "status": "success", 17:32:05 DEBUG --- status code --- 17:32:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:32:05 DEBUG "data": { 17:32:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:05 DEBUG ----- output ----- 17:32:05 DEBUG "resultType": "vector", 17:32:05 DEBUG --- http response --- 17:32:05 DEBUG { 17:32:05 DEBUG "result": [ 17:32:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:32:05 DEBUG "status": "success", 17:32:05 DEBUG { 17:32:05 DEBUG ----- output ----- 17:32:05 DEBUG "data": { 17:32:05 DEBUG "metric": { 17:32:05 DEBUG { 17:32:05 DEBUG "resultType": "vector", 17:32:05 DEBUG "pod": "ds-idrepo-0" 17:32:05 DEBUG "status": "success", 17:32:05 DEBUG "data": { 17:32:05 DEBUG "resultType": "vector", 17:32:05 DEBUG "result": [ 17:32:05 DEBUG }, 17:32:05 DEBUG { 17:32:05 DEBUG "metric": { 17:32:05 DEBUG "value": [ 17:32:05 DEBUG "pod": "ds-cts-0" 17:32:05 DEBUG 17:32:05 DEBUG "result": [ 17:32:05 DEBUG }, 17:32:05 DEBUG 17:32:05 DEBUG { 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 17:32:05 DEBUG "metric": { 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG "pod": "ds-cts-0" 17:32:05 DEBUG [...] 17:32:05 DEBUG 17:32:05 DEBUG }, 17:32:05 DEBUG 17:32:05 DEBUG [...] 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG "pod": "ds-idrepo-2" 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG }, 17:32:05 DEBUG "pod": "ds-cts-2" 17:32:05 DEBUG [...] 17:32:05 DEBUG "value": [ 17:32:05 DEBUG }, 17:32:05 DEBUG 17:32:05 DEBUG 1713544325, 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 17:32:05 DEBUG "0" 17:32:05 DEBUG 1713544325, 17:32:05 DEBUG 17:32:05 DEBUG "pod": "ds-cts-2" 17:32:05 DEBUG "0.12174469333333333" 17:32:05 DEBUG ] 17:32:05 DEBUG }, 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG "value": [ 17:32:05 DEBUG } 17:32:05 DEBUG ] 17:32:05 DEBUG 1713544325, 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG } 17:32:05 DEBUG } 17:32:05 DEBUG "0" 17:32:05 DEBUG } 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG } 17:32:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544325" 17:32:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544325" 17:32:05 INFO [http_cmd]: http status code OK 17:32:05 DEBUG --- status code --- 17:32:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:05 DEBUG --- http response --- 17:32:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 17:32:05 DEBUG ----- output ----- 17:32:05 DEBUG { 17:32:05 DEBUG "status": "success", 17:32:05 DEBUG "data": { 17:32:05 DEBUG "resultType": "vector", 17:32:05 DEBUG "result": [ 17:32:05 DEBUG { 17:32:05 DEBUG "metric": { 17:32:05 DEBUG "pod": "ds-idrepo-0" 17:32:05 DEBUG }, 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG [...] 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG "pod": "ds-cts-1" 17:32:05 DEBUG }, 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 1713544325, 17:32:05 DEBUG "31869010.13517328" 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG } 17:32:05 INFO [http_cmd]: http status code OK 17:32:05 DEBUG --- status code --- 17:32:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:05 DEBUG --- http response --- 17:32:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:32:05 DEBUG ----- output ----- 17:32:05 DEBUG { 17:32:05 DEBUG "status": "success", 17:32:05 DEBUG "data": { 17:32:05 DEBUG "resultType": "vector", 17:32:05 DEBUG "result": [ 17:32:05 DEBUG { 17:32:05 DEBUG "metric": { 17:32:05 DEBUG "instance": "10.142.0.107:9100" 17:32:05 DEBUG }, 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG [...] 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG "instance": "10.142.0.43:9100" 17:32:05 DEBUG }, 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 1713544325, 17:32:05 DEBUG "75786835.56666666" 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG } 17:32:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544325" 17:32:05 INFO [http_cmd]: http status code OK 17:32:05 DEBUG --- status code --- 17:32:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:05 DEBUG --- http response --- 17:32:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:32:05 DEBUG ----- output ----- 17:32:05 DEBUG { 17:32:05 DEBUG "status": "success", 17:32:05 DEBUG "data": { 17:32:05 DEBUG "resultType": "vector", 17:32:05 DEBUG "result": [ 17:32:05 DEBUG { 17:32:05 DEBUG "metric": { 17:32:05 DEBUG "instance": "10.142.0.107:9100" 17:32:05 DEBUG }, 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG [...] 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG 17:32:05 DEBUG "instance": "10.142.0.43:9100" 17:32:05 DEBUG }, 17:32:05 DEBUG "value": [ 17:32:05 DEBUG 1713544325, 17:32:05 DEBUG "75779692.7" 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG ] 17:32:05 DEBUG } 17:32:05 DEBUG } 17:32:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544326" 17:32:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544326" 17:32:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544326" 17:32:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544326" 17:32:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544326" 17:32:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544326" 17:32:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544326" 17:32:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544326" 17:32:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544326" 17:32:06 INFO [http_cmd]: http status code OK 17:32:06 INFO [http_cmd]: http status code OK 17:32:06 DEBUG --- status code --- 17:32:06 INFO [http_cmd]: http status code OK 17:32:06 INFO [http_cmd]: http status code OK 17:32:06 DEBUG --- status code --- 17:32:06 INFO [http_cmd]: http status code OK 17:32:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:06 INFO [http_cmd]: http status code OK 17:32:06 DEBUG --- status code --- 17:32:06 INFO [http_cmd]: http status code OK 17:32:06 DEBUG --- status code --- 17:32:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:06 DEBUG --- status code --- 17:32:06 DEBUG --- status code --- 17:32:06 DEBUG --- http response --- 17:32:06 INFO [http_cmd]: http status code OK 17:32:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:06 DEBUG --- status code --- 17:32:06 DEBUG --- http response --- 17:32:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:06 DEBUG --- http response --- 17:32:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 17:32:06 DEBUG --- http response --- 17:32:06 INFO [http_cmd]: http status code OK 17:32:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:06 DEBUG --- http response --- 17:32:06 DEBUG --- http response --- 17:32:06 DEBUG --- http response --- 17:32:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:32:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:32:06 DEBUG --- status code --- 17:32:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:32:06 DEBUG ----- output ----- 17:32:06 DEBUG { 17:32:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:32:06 DEBUG ----- output ----- 17:32:06 DEBUG ----- output ----- 17:32:06 DEBUG ----- output ----- 17:32:06 DEBUG "status": "success", 17:32:06 DEBUG "data": { 17:32:06 DEBUG --- status code --- 17:32:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:32:06 DEBUG ----- output ----- 17:32:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:32:06 DEBUG { 17:32:06 DEBUG { 17:32:06 DEBUG { 17:32:06 DEBUG "status": "success", 17:32:06 DEBUG --- http response --- 17:32:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:32:06 DEBUG ----- output ----- 17:32:06 DEBUG { 17:32:06 DEBUG ----- output ----- 17:32:06 DEBUG "status": "success", 17:32:06 DEBUG "status": "success", 17:32:06 DEBUG "data": { 17:32:06 DEBUG "resultType": "vector", 17:32:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:32:06 DEBUG --- http response --- 17:32:06 DEBUG "status": "success", 17:32:06 DEBUG { 17:32:06 DEBUG { 17:32:06 DEBUG "data": { 17:32:06 DEBUG "data": { 17:32:06 DEBUG "resultType": "vector", 17:32:06 DEBUG "result": [ 17:32:06 DEBUG ----- output ----- 17:32:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:32:06 DEBUG "data": { 17:32:06 DEBUG "status": "success", 17:32:06 DEBUG "status": "success", 17:32:06 DEBUG "data": { 17:32:06 DEBUG "resultType": "vector", 17:32:06 DEBUG "resultType": "vector", 17:32:06 DEBUG { 17:32:06 DEBUG { 17:32:06 DEBUG ----- output ----- 17:32:06 DEBUG "resultType": "vector", 17:32:06 DEBUG "data": { 17:32:06 DEBUG "result": [ 17:32:06 DEBUG { 17:32:06 DEBUG "result": [ 17:32:06 DEBUG "result": [ 17:32:06 DEBUG "metric": { 17:32:06 DEBUG "status": "success", 17:32:06 DEBUG { 17:32:06 DEBUG "result": [ 17:32:06 DEBUG "resultType": "vector", 17:32:06 DEBUG "result": [ 17:32:06 DEBUG "metric": { 17:32:06 DEBUG { 17:32:06 DEBUG { 17:32:06 DEBUG "pod": "ds-idrepo-0" 17:32:06 DEBUG "data": { 17:32:06 DEBUG "status": "success", 17:32:06 DEBUG { 17:32:06 DEBUG "resultType": "vector", 17:32:06 DEBUG { 17:32:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:32:06 DEBUG }, 17:32:06 DEBUG "metric": { 17:32:06 DEBUG }, 17:32:06 DEBUG "resultType": "vector", 17:32:06 DEBUG "result": [ 17:32:06 DEBUG { 17:32:06 DEBUG "metric": { 17:32:06 DEBUG "instance": "10.142.0.107:9100" 17:32:06 DEBUG "value": [ 17:32:06 DEBUG 17:32:06 DEBUG "value": [ 17:32:06 DEBUG "pod": "ds-idrepo-0" 17:32:06 DEBUG }, 17:32:06 DEBUG "metric": { 17:32:06 DEBUG "result": [ 17:32:06 DEBUG 17:32:06 DEBUG "data": { 17:32:06 DEBUG 17:32:06 DEBUG }, 17:32:06 DEBUG "value": [ 17:32:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:32:06 DEBUG { 17:32:06 DEBUG 17:32:06 DEBUG "metric": { 17:32:06 DEBUG "resultType": "vector", 17:32:06 DEBUG "metric": { 17:32:06 DEBUG 17:32:06 DEBUG "value": [ 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG "metric": { 17:32:06 DEBUG [...] 17:32:06 DEBUG "pod": "ds-cts-0" 17:32:06 DEBUG "result": [ 17:32:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:32:06 DEBUG }, 17:32:06 DEBUG }, 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG "pod": "ds-idrepo-0" 17:32:06 DEBUG 17:32:06 DEBUG }, 17:32:06 DEBUG { 17:32:06 DEBUG 17:32:06 DEBUG "value": [ 17:32:06 DEBUG "value": [ 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG }, 17:32:06 DEBUG 17:32:06 DEBUG "value": [ 17:32:06 DEBUG "metric": { 17:32:06 DEBUG [...] 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG [...] 17:32:06 DEBUG [...] 17:32:06 DEBUG "value": [ 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:32:06 DEBUG 17:32:06 DEBUG }, 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG }, 17:32:06 DEBUG 17:32:06 DEBUG "value": [ 17:32:06 DEBUG 17:32:06 DEBUG [...] 17:32:06 DEBUG [...] 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG "value": [ 17:32:06 DEBUG [...] 17:32:06 DEBUG 17:32:06 DEBUG "pod": "ds-idrepo-2" 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG "pod": "ds-cts-1" 17:32:06 DEBUG "instance": "10.142.0.43:9100" 17:32:06 DEBUG [...] 17:32:06 DEBUG 1713544326, 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG }, 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG }, 17:32:06 DEBUG }, 17:32:06 DEBUG 17:32:06 DEBUG "2489.733333333333" 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG "value": [ 17:32:06 DEBUG 17:32:06 DEBUG 17:32:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:32:06 DEBUG "value": [ 17:32:06 DEBUG 17:32:06 DEBUG ] 17:32:06 DEBUG 17:32:06 DEBUG [...] 17:32:06 DEBUG 1713544326, 17:32:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:32:06 DEBUG }, 17:32:06 DEBUG 1713544326, 17:32:06 DEBUG "value": [ 17:32:06 DEBUG 17:32:06 DEBUG } 17:32:06 DEBUG "pod": "ds-cts-2" 17:32:06 DEBUG 17:32:06 DEBUG "0" 17:32:06 DEBUG }, 17:32:06 DEBUG "value": [ 17:32:06 DEBUG "0.8328999999999581" 17:32:06 DEBUG 1713544326, 17:32:06 DEBUG "pod": "ds-idrepo-2" 17:32:06 DEBUG ] 17:32:06 DEBUG }, 17:32:06 DEBUG 17:32:06 DEBUG ] 17:32:06 DEBUG "value": [ 17:32:06 DEBUG 1713544326, 17:32:06 DEBUG ] 17:32:06 DEBUG "12702562.82180754" 17:32:06 DEBUG }, 17:32:06 DEBUG } 17:32:06 DEBUG "value": [ 17:32:06 DEBUG 17:32:06 DEBUG } 17:32:06 DEBUG 1713544326, 17:32:06 DEBUG "7.951686804966192" 17:32:06 DEBUG } 17:32:06 DEBUG ] 17:32:06 DEBUG "value": [ 17:32:06 DEBUG } 17:32:06 DEBUG 1713544326, 17:32:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:32:06 DEBUG ] 17:32:06 DEBUG "2478.733333333333" 17:32:06 DEBUG ] 17:32:06 DEBUG ] 17:32:06 DEBUG } 17:32:06 DEBUG 1713544326, 17:32:06 DEBUG "310.73333333333335" 17:32:06 DEBUG }, 17:32:06 DEBUG } 17:32:06 DEBUG ] 17:32:06 DEBUG } 17:32:06 DEBUG } 17:32:06 DEBUG ] 17:32:06 DEBUG "0" 17:32:06 DEBUG ] 17:32:06 DEBUG "value": [ 17:32:06 DEBUG } 17:32:06 DEBUG } 17:32:06 DEBUG ] 17:32:06 DEBUG } 17:32:06 DEBUG } 17:32:06 DEBUG ] 17:32:06 DEBUG } 17:32:06 DEBUG 1713544326, 17:32:06 DEBUG ] 17:32:06 DEBUG } 17:32:06 DEBUG } 17:32:06 DEBUG } 17:32:06 DEBUG ] 17:32:06 DEBUG "0.023" 17:32:06 DEBUG } 17:32:06 DEBUG } 17:32:06 DEBUG ] 17:32:06 DEBUG } 17:32:06 DEBUG ] 17:32:06 DEBUG } 17:32:06 DEBUG } 17:32:06 DEBUG } 17:32:06 DEBUG } 17:32:06 DEBUG } 17:32:06 DEBUG ] 17:32:06 DEBUG } 17:32:06 DEBUG } 17:32:06 INFO 17:32:06 INFO [loop_until]: kubectl --namespace=xlou top pods 17:32:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:32:06 INFO [loop_until]: OK (rc = 0) 17:32:06 DEBUG --- stdout --- 17:32:06 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 16835m 8902Mi am-6fd554f8f7-btd4s 16374m 8931Mi am-6fd554f8f7-wxcsj 18502m 8943Mi ds-cts-0 20901m 12842Mi ds-cts-1 21137m 12904Mi ds-cts-2 21567m 13015Mi ds-idrepo-0 7098m 3422Mi ds-idrepo-1 6789m 3388Mi ds-idrepo-2 6904m 3460Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7923m 1169Mi 17:32:06 DEBUG --- stderr --- 17:32:06 DEBUG 17:32:06 INFO 17:32:06 INFO [loop_until]: kubectl --namespace=xlou top node 17:32:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:32:06 INFO [loop_until]: OK (rc = 0) 17:32:06 DEBUG --- stdout --- 17:32:06 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6178Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7458m 24% 6999Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 49m 0% 2683Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17077m 57% 12128Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 21377m 71% 16090Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5221Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17482m 58% 11829Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16753m 56% 3015Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19144m 64% 11166Mi 10% gke-xlou-cdm-ds-0f601751-j488 7562m 25% 6686Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21487m 71% 17950Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 7494m 25% 7213Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21744m 72% 15518Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7913m 49% 4932Mi 8% 17:32:06 DEBUG --- stderr --- 17:32:06 DEBUG 17:33:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544383" 17:33:03 ERROR [http_cmd]: ERROR 17:33:03 ERROR --- status code --- 17:33:03 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:33:03 ERROR --- http response --- 17:33:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:33:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:33:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544383" 17:33:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544383" 17:33:03 INFO [http_cmd]: http status code OK 17:33:03 DEBUG --- status code --- 17:33:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:03 DEBUG --- http response --- 17:33:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:33:03 INFO [http_cmd]: http status code OK 17:33:03 DEBUG --- status code --- 17:33:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:03 DEBUG --- http response --- 17:33:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:33:03 DEBUG ----- output ----- 17:33:03 DEBUG { 17:33:03 DEBUG "status": "success", 17:33:03 DEBUG "data": { 17:33:03 DEBUG "resultType": "vector", 17:33:03 DEBUG "result": [ 17:33:03 DEBUG { 17:33:03 DEBUG "metric": { 17:33:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:33:03 DEBUG }, 17:33:03 DEBUG "value": [ 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG [...] 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:33:03 DEBUG }, 17:33:03 DEBUG "value": [ 17:33:03 DEBUG 1713544383, 17:33:03 DEBUG "2339.3" 17:33:03 DEBUG ] 17:33:03 DEBUG } 17:33:03 DEBUG ] 17:33:03 DEBUG } 17:33:03 DEBUG } 17:33:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544383" 17:33:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544383" 17:33:03 INFO [http_cmd]: http status code OK 17:33:03 INFO [http_cmd]: http status code OK 17:33:03 DEBUG --- status code --- 17:33:03 DEBUG --- status code --- 17:33:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:03 DEBUG --- http response --- 17:33:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:33:03 DEBUG ----- output ----- 17:33:03 DEBUG { 17:33:03 DEBUG "status": "success", 17:33:03 DEBUG "data": { 17:33:03 DEBUG "resultType": "vector", 17:33:03 DEBUG "result": [ 17:33:03 DEBUG { 17:33:03 DEBUG "metric": { 17:33:03 DEBUG "pod": "ds-idrepo-0" 17:33:03 DEBUG }, 17:33:03 DEBUG "value": [ 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG [...] 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG "pod": "ds-cts-2" 17:33:03 DEBUG }, 17:33:03 DEBUG "value": [ 17:33:03 DEBUG 1713544383, 17:33:03 DEBUG "0" 17:33:03 DEBUG ] 17:33:03 DEBUG } 17:33:03 DEBUG ] 17:33:03 DEBUG } 17:33:03 DEBUG } 17:33:03 DEBUG --- http response --- 17:33:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:33:03 DEBUG ----- output ----- 17:33:03 DEBUG { 17:33:03 DEBUG "status": "success", 17:33:03 DEBUG "data": { 17:33:03 DEBUG "resultType": "vector", 17:33:03 DEBUG "result": [ 17:33:03 DEBUG { 17:33:03 DEBUG "metric": { 17:33:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:33:03 DEBUG }, 17:33:03 DEBUG "value": [ 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG [...] 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:33:03 DEBUG }, 17:33:03 DEBUG "value": [ 17:33:03 DEBUG 1713544383, 17:33:03 DEBUG "2505.733333333333" 17:33:03 DEBUG ] 17:33:03 DEBUG } 17:33:03 DEBUG ] 17:33:03 DEBUG } 17:33:03 DEBUG } 17:33:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544383" 17:33:03 INFO [http_cmd]: http status code OK 17:33:03 DEBUG --- status code --- 17:33:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:03 DEBUG --- http response --- 17:33:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:33:03 DEBUG ----- output ----- 17:33:03 DEBUG { 17:33:03 DEBUG "status": "success", 17:33:03 DEBUG "data": { 17:33:03 DEBUG "resultType": "vector", 17:33:03 DEBUG "result": [ 17:33:03 DEBUG { 17:33:03 DEBUG "metric": { 17:33:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:33:03 DEBUG }, 17:33:03 DEBUG "value": [ 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG [...] 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:33:03 DEBUG }, 17:33:03 DEBUG "value": [ 17:33:03 DEBUG 1713544383, 17:33:03 DEBUG "0" 17:33:03 DEBUG ] 17:33:03 DEBUG } 17:33:03 DEBUG ] 17:33:03 DEBUG } 17:33:03 DEBUG } 17:33:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544383" 17:33:03 INFO [http_cmd]: http status code OK 17:33:03 DEBUG --- status code --- 17:33:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:03 DEBUG --- http response --- 17:33:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:33:03 DEBUG ----- output ----- 17:33:03 DEBUG { 17:33:03 DEBUG "status": "success", 17:33:03 DEBUG "data": { 17:33:03 DEBUG "resultType": "vector", 17:33:03 DEBUG "result": [ 17:33:03 DEBUG { 17:33:03 DEBUG "metric": { 17:33:03 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:33:03 DEBUG }, 17:33:03 DEBUG "value": [ 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG [...] 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG 17:33:03 DEBUG "pod": "ds-idrepo-2" 17:33:03 DEBUG }, 17:33:03 DEBUG "value": [ 17:33:03 DEBUG 1713544383, 17:33:03 DEBUG "129612522.39509664" 17:33:03 DEBUG ] 17:33:03 DEBUG } 17:33:03 DEBUG ] 17:33:03 DEBUG } 17:33:03 DEBUG } 17:33:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544384" 17:33:04 INFO [http_cmd]: http status code OK 17:33:04 DEBUG --- status code --- 17:33:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:04 DEBUG --- http response --- 17:33:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:33:04 DEBUG ----- output ----- 17:33:04 DEBUG { 17:33:04 DEBUG "status": "success", 17:33:04 DEBUG "data": { 17:33:04 DEBUG "resultType": "vector", 17:33:04 DEBUG "result": [ 17:33:04 DEBUG { 17:33:04 DEBUG "metric": { 17:33:04 DEBUG "instance": "10.142.0.107:9100" 17:33:04 DEBUG }, 17:33:04 DEBUG "value": [ 17:33:04 DEBUG 17:33:04 DEBUG 17:33:04 DEBUG 17:33:04 DEBUG [...] 17:33:04 DEBUG 17:33:04 DEBUG 17:33:04 DEBUG 17:33:04 DEBUG "instance": "10.142.0.43:9100" 17:33:04 DEBUG }, 17:33:04 DEBUG "value": [ 17:33:04 DEBUG 1713544384, 17:33:04 DEBUG "0.34500000000000075" 17:33:04 DEBUG ] 17:33:04 DEBUG } 17:33:04 DEBUG ] 17:33:04 DEBUG } 17:33:04 DEBUG } 17:33:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544384" 17:33:04 INFO [http_cmd]: http status code OK 17:33:04 DEBUG --- status code --- 17:33:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:04 DEBUG --- http response --- 17:33:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:33:04 DEBUG ----- output ----- 17:33:04 DEBUG { 17:33:04 DEBUG "status": "success", 17:33:04 DEBUG "data": { 17:33:04 DEBUG "resultType": "vector", 17:33:04 DEBUG "result": [ 17:33:04 DEBUG { 17:33:04 DEBUG "metric": { 17:33:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:33:04 DEBUG }, 17:33:04 DEBUG "value": [ 17:33:04 DEBUG 17:33:04 DEBUG 17:33:04 DEBUG 17:33:04 DEBUG [...] 17:33:04 DEBUG 17:33:04 DEBUG 17:33:04 DEBUG 17:33:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:33:04 DEBUG }, 17:33:04 DEBUG "value": [ 17:33:04 DEBUG 1713544384, 17:33:04 DEBUG "4076870.5" 17:33:04 DEBUG ] 17:33:04 DEBUG } 17:33:04 DEBUG ] 17:33:04 DEBUG } 17:33:04 DEBUG } 17:33:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544384" 17:33:05 INFO [http_cmd]: http status code OK 17:33:05 DEBUG --- status code --- 17:33:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:05 DEBUG --- http response --- 17:33:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:33:05 DEBUG ----- output ----- 17:33:05 DEBUG { 17:33:05 DEBUG "status": "success", 17:33:05 DEBUG "data": { 17:33:05 DEBUG "resultType": "vector", 17:33:05 DEBUG "result": [ 17:33:05 DEBUG { 17:33:05 DEBUG "metric": { 17:33:05 DEBUG "instance": "10.142.0.107:9100" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG [...] 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG "instance": "10.142.0.43:9100" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 1713544384, 17:33:05 DEBUG "1716119469056" 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG } 17:33:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544385" 17:33:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544385" 17:33:05 INFO [http_cmd]: http status code OK 17:33:05 DEBUG --- status code --- 17:33:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:05 DEBUG --- http response --- 17:33:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:33:05 DEBUG ----- output ----- 17:33:05 DEBUG { 17:33:05 DEBUG "status": "success", 17:33:05 DEBUG "data": { 17:33:05 DEBUG "resultType": "vector", 17:33:05 DEBUG "result": [ 17:33:05 DEBUG { 17:33:05 DEBUG "metric": { 17:33:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG [...] 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 1713544385, 17:33:05 DEBUG "1327349760" 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG } 17:33:05 INFO [http_cmd]: http status code OK 17:33:05 DEBUG --- status code --- 17:33:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:05 DEBUG --- http response --- 17:33:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:33:05 DEBUG ----- output ----- 17:33:05 DEBUG { 17:33:05 DEBUG "status": "success", 17:33:05 DEBUG "data": { 17:33:05 DEBUG "resultType": "vector", 17:33:05 DEBUG "result": [ 17:33:05 DEBUG { 17:33:05 DEBUG "metric": { 17:33:05 DEBUG "pod": "ds-idrepo-0" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG [...] 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG "pod": "ds-idrepo-2" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 1713544385, 17:33:05 DEBUG "0" 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG } 17:33:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544385" 17:33:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544385" 17:33:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544385" 17:33:05 INFO [http_cmd]: http status code OK 17:33:05 DEBUG --- status code --- 17:33:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:05 DEBUG --- http response --- 17:33:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:33:05 DEBUG ----- output ----- 17:33:05 DEBUG { 17:33:05 DEBUG "status": "success", 17:33:05 DEBUG "data": { 17:33:05 DEBUG "resultType": "vector", 17:33:05 DEBUG "result": [ 17:33:05 DEBUG { 17:33:05 DEBUG "metric": { 17:33:05 DEBUG "pod": "ds-idrepo-0" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG [...] 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG "pod": "ds-idrepo-2" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 1713544385, 17:33:05 DEBUG "0" 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG } 17:33:05 INFO [http_cmd]: http status code OK 17:33:05 DEBUG --- status code --- 17:33:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:05 DEBUG --- http response --- 17:33:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:33:05 DEBUG ----- output ----- 17:33:05 DEBUG { 17:33:05 DEBUG "status": "success", 17:33:05 DEBUG "data": { 17:33:05 DEBUG "resultType": "vector", 17:33:05 DEBUG "result": [ 17:33:05 DEBUG { 17:33:05 DEBUG "metric": { 17:33:05 DEBUG "pod": "ds-cts-0" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG [...] 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG "pod": "ds-cts-2" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 1713544385, 17:33:05 DEBUG "0" 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG } 17:33:05 INFO [http_cmd]: http status code OK 17:33:05 DEBUG --- status code --- 17:33:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:05 DEBUG --- http response --- 17:33:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:33:05 DEBUG ----- output ----- 17:33:05 DEBUG { 17:33:05 DEBUG "status": "success", 17:33:05 DEBUG "data": { 17:33:05 DEBUG "resultType": "vector", 17:33:05 DEBUG "result": [ 17:33:05 DEBUG { 17:33:05 DEBUG "metric": { 17:33:05 DEBUG "pod": "ds-cts-0" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG [...] 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG "pod": "ds-cts-2" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 1713544385, 17:33:05 DEBUG "0.19179999999999997" 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG } 17:33:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544385" 17:33:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544385" 17:33:05 INFO [http_cmd]: http status code OK 17:33:05 DEBUG --- status code --- 17:33:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:05 DEBUG --- http response --- 17:33:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:33:05 DEBUG ----- output ----- 17:33:05 DEBUG { 17:33:05 DEBUG "status": "success", 17:33:05 DEBUG "data": { 17:33:05 DEBUG "resultType": "vector", 17:33:05 DEBUG "result": [ 17:33:05 DEBUG { 17:33:05 DEBUG "metric": { 17:33:05 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG [...] 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG "pod": "ds-idrepo-2" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 1713544385, 17:33:05 DEBUG "4103.724658180104" 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG } 17:33:05 INFO [http_cmd]: http status code OK 17:33:05 DEBUG --- status code --- 17:33:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:05 DEBUG --- http response --- 17:33:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:33:05 DEBUG ----- output ----- 17:33:05 DEBUG { 17:33:05 DEBUG "status": "success", 17:33:05 DEBUG "data": { 17:33:05 DEBUG "resultType": "vector", 17:33:05 DEBUG "result": [ 17:33:05 DEBUG { 17:33:05 DEBUG "metric": { 17:33:05 DEBUG "instance": "10.142.0.107:9100" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG [...] 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG "instance": "10.142.0.43:9100" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 1713544385, 17:33:05 DEBUG "75748661.7" 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG } 17:33:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544385" 17:33:05 INFO [http_cmd]: http status code OK 17:33:05 DEBUG --- status code --- 17:33:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:05 DEBUG --- http response --- 17:33:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:33:05 DEBUG ----- output ----- 17:33:05 DEBUG { 17:33:05 DEBUG "status": "success", 17:33:05 DEBUG "data": { 17:33:05 DEBUG "resultType": "vector", 17:33:05 DEBUG "result": [ 17:33:05 DEBUG { 17:33:05 DEBUG "metric": { 17:33:05 DEBUG "instance": "10.142.0.107:9100" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG [...] 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG 17:33:05 DEBUG "instance": "10.142.0.43:9100" 17:33:05 DEBUG }, 17:33:05 DEBUG "value": [ 17:33:05 DEBUG 1713544385, 17:33:05 DEBUG "75742226.30000001" 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG ] 17:33:05 DEBUG } 17:33:05 DEBUG } 17:33:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544386" 17:33:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544386" 17:33:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544386" 17:33:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544386" 17:33:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544386" 17:33:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544386" 17:33:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544386" 17:33:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544386" 17:33:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544386" 17:33:06 INFO [http_cmd]: http status code OK 17:33:06 DEBUG --- status code --- 17:33:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:06 DEBUG --- http response --- 17:33:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:33:06 DEBUG ----- output ----- 17:33:06 DEBUG { 17:33:06 INFO [http_cmd]: http status code OK 17:33:06 DEBUG "status": "success", 17:33:06 DEBUG --- status code --- 17:33:06 DEBUG "data": { 17:33:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:06 DEBUG "resultType": "vector", 17:33:06 DEBUG --- http response --- 17:33:06 DEBUG "result": [ 17:33:06 INFO [http_cmd]: http status code OK 17:33:06 INFO [http_cmd]: http status code OK 17:33:06 INFO [http_cmd]: http status code OK 17:33:06 DEBUG { 17:33:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:33:06 INFO [http_cmd]: http status code OK 17:33:06 INFO [http_cmd]: http status code OK 17:33:06 DEBUG --- status code --- 17:33:06 DEBUG --- status code --- 17:33:06 DEBUG --- status code --- 17:33:06 DEBUG "metric": { 17:33:06 DEBUG ----- output ----- 17:33:06 DEBUG --- status code --- 17:33:06 DEBUG --- status code --- 17:33:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:06 INFO [http_cmd]: http status code OK 17:33:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:06 INFO [http_cmd]: http status code OK 17:33:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:33:06 DEBUG }, 17:33:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:06 DEBUG --- http response --- 17:33:06 DEBUG --- http response --- 17:33:06 DEBUG --- status code --- 17:33:06 DEBUG --- http response --- 17:33:06 DEBUG --- status code --- 17:33:06 DEBUG { 17:33:06 DEBUG "value": [ 17:33:06 DEBUG 17:33:06 DEBUG --- http response --- 17:33:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:33:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:33:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:33:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:33:06 DEBUG "status": "success", 17:33:06 DEBUG 17:33:06 DEBUG --- http response --- 17:33:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:33:06 DEBUG ----- output ----- 17:33:06 DEBUG ----- output ----- 17:33:06 DEBUG --- http response --- 17:33:06 DEBUG ----- output ----- 17:33:06 DEBUG --- http response --- 17:33:06 DEBUG "data": { 17:33:06 DEBUG 17:33:06 DEBUG [...] 17:33:06 DEBUG ----- output ----- 17:33:06 DEBUG { 17:33:06 DEBUG { 17:33:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:33:06 DEBUG { 17:33:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:33:06 DEBUG "resultType": "vector", 17:33:06 DEBUG 17:33:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:33:06 DEBUG ----- output ----- 17:33:06 DEBUG "status": "success", 17:33:06 DEBUG "status": "success", 17:33:06 DEBUG ----- output ----- 17:33:06 DEBUG "status": "success", 17:33:06 DEBUG "data": { 17:33:06 DEBUG "result": [ 17:33:06 DEBUG 17:33:06 DEBUG { 17:33:06 DEBUG { 17:33:06 DEBUG "data": { 17:33:06 DEBUG "data": { 17:33:06 DEBUG { 17:33:06 DEBUG "resultType": "vector", 17:33:06 DEBUG { 17:33:06 DEBUG ----- output ----- 17:33:06 DEBUG 17:33:06 DEBUG "status": "success", 17:33:06 DEBUG "status": "success", 17:33:06 DEBUG "data": { 17:33:06 DEBUG "resultType": "vector", 17:33:06 DEBUG "status": "success", 17:33:06 DEBUG "result": [ 17:33:06 DEBUG "metric": { 17:33:06 DEBUG { 17:33:06 DEBUG "status": "success", 17:33:06 DEBUG "data": { 17:33:06 DEBUG "resultType": "vector", 17:33:06 DEBUG "result": [ 17:33:06 DEBUG "data": { 17:33:06 DEBUG { 17:33:06 DEBUG "resultType": "vector", 17:33:06 DEBUG "pod": "ds-idrepo-0" 17:33:06 DEBUG "data": { 17:33:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:33:06 DEBUG "resultType": "vector", 17:33:06 DEBUG "result": [ 17:33:06 DEBUG { 17:33:06 DEBUG "resultType": "vector", 17:33:06 DEBUG "metric": { 17:33:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:33:06 DEBUG }, 17:33:06 DEBUG "resultType": "vector", 17:33:06 DEBUG }, 17:33:06 DEBUG "result": [ 17:33:06 DEBUG { 17:33:06 DEBUG "metric": { 17:33:06 DEBUG "result": [ 17:33:06 DEBUG "result": [ 17:33:06 DEBUG }, 17:33:06 DEBUG "value": [ 17:33:06 DEBUG "result": [ 17:33:06 DEBUG "value": [ 17:33:06 DEBUG { 17:33:06 DEBUG "metric": { 17:33:06 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:33:06 DEBUG }, 17:33:06 DEBUG { 17:33:06 DEBUG "value": [ 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG 1713544386, 17:33:06 DEBUG "metric": { 17:33:06 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:33:06 DEBUG "value": [ 17:33:06 DEBUG { 17:33:06 DEBUG "metric": { 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG "2505.3" 17:33:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:33:06 DEBUG }, 17:33:06 DEBUG 17:33:06 DEBUG "metric": { 17:33:06 DEBUG "instance": "10.142.0.107:9100" 17:33:06 DEBUG 17:33:06 DEBUG [...] 17:33:06 DEBUG 17:33:06 DEBUG ] 17:33:06 DEBUG }, 17:33:06 DEBUG "value": [ 17:33:06 DEBUG 17:33:06 DEBUG "pod": "ds-cts-0" 17:33:06 DEBUG }, 17:33:06 DEBUG [...] 17:33:06 DEBUG { 17:33:06 DEBUG "metric": { 17:33:06 DEBUG } 17:33:06 DEBUG ] 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG }, 17:33:06 DEBUG "value": [ 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:33:06 DEBUG }, 17:33:06 DEBUG } 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG [...] 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:33:06 DEBUG "value": [ 17:33:06 DEBUG } 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG [...] 17:33:06 DEBUG "value": [ 17:33:06 DEBUG 17:33:06 DEBUG }, 17:33:06 DEBUG 17:33:06 DEBUG "value": [ 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG "value": [ 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG 1713544386, 17:33:06 DEBUG "pod": "ds-idrepo-2" 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG [...] 17:33:06 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:33:06 DEBUG 17:33:06 DEBUG "2504.4333333333334" 17:33:06 DEBUG [...] 17:33:06 DEBUG }, 17:33:06 DEBUG }, 17:33:06 DEBUG "value": [ 17:33:06 DEBUG [...] 17:33:06 DEBUG }, 17:33:06 DEBUG [...] 17:33:06 DEBUG ] 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG 1713544386, 17:33:06 DEBUG 17:33:06 DEBUG "value": [ 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG 17:33:06 DEBUG "value": [ 17:33:06 DEBUG 17:33:06 DEBUG "0" 17:33:06 DEBUG 17:33:06 DEBUG 1713544386, 17:33:06 DEBUG 17:33:06 DEBUG } 17:33:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:33:06 DEBUG 1713544386, 17:33:06 DEBUG 17:33:06 DEBUG ] 17:33:06 DEBUG 17:33:06 DEBUG "0.0019517897365535681" 17:33:06 DEBUG "pod": "ds-cts-2" 17:33:06 DEBUG }, 17:33:06 DEBUG }, 17:33:06 DEBUG "value": [ 17:33:06 DEBUG 1713544386, 17:33:06 DEBUG } 17:33:06 DEBUG "instance": "10.142.0.43:9100" 17:33:06 DEBUG ] 17:33:06 DEBUG "8.03136297693759" 17:33:06 DEBUG ] 17:33:06 DEBUG ] 17:33:06 DEBUG ] 17:33:06 DEBUG } 17:33:06 DEBUG }, 17:33:06 DEBUG } 17:33:06 DEBUG } 17:33:06 DEBUG "value": [ 17:33:06 DEBUG } 17:33:06 DEBUG } 17:33:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:33:06 DEBUG "value": [ 17:33:06 DEBUG 1713544386, 17:33:06 DEBUG ] 17:33:06 DEBUG 1713544386, 17:33:06 DEBUG "0" 17:33:06 DEBUG } 17:33:06 DEBUG }, 17:33:06 DEBUG "0.8082666666666929" 17:33:06 DEBUG ] 17:33:06 DEBUG } 17:33:06 DEBUG } 17:33:06 DEBUG ] 17:33:06 DEBUG "value": [ 17:33:06 DEBUG ] 17:33:06 DEBUG } 17:33:06 DEBUG } 17:33:06 DEBUG } 17:33:06 DEBUG 1713544386, 17:33:06 DEBUG } 17:33:06 DEBUG "323.3" 17:33:06 DEBUG ] 17:33:06 DEBUG "0.023" 17:33:06 DEBUG ] 17:33:06 DEBUG ] 17:33:06 DEBUG } 17:33:06 DEBUG ] 17:33:06 DEBUG } 17:33:06 DEBUG ] 17:33:06 DEBUG } 17:33:06 DEBUG } 17:33:06 DEBUG } 17:33:06 DEBUG } 17:33:06 DEBUG ] 17:33:06 DEBUG } 17:33:06 DEBUG } 17:33:06 DEBUG } 17:33:06 DEBUG } 17:33:06 INFO 17:33:06 INFO [loop_until]: kubectl --namespace=xlou top pods 17:33:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:33:06 INFO [loop_until]: OK (rc = 0) 17:33:06 DEBUG --- stdout --- 17:33:06 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17442m 8954Mi am-6fd554f8f7-btd4s 17437m 8979Mi am-6fd554f8f7-wxcsj 19814m 8961Mi ds-cts-0 21136m 13270Mi ds-cts-1 21582m 12627Mi ds-cts-2 21196m 13177Mi ds-idrepo-0 6820m 3423Mi ds-idrepo-1 6994m 3387Mi ds-idrepo-2 6855m 3460Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8005m 1173Mi 17:33:06 DEBUG --- stderr --- 17:33:06 DEBUG 17:33:06 INFO 17:33:06 INFO [loop_until]: kubectl --namespace=xlou top node 17:33:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:33:06 INFO [loop_until]: OK (rc = 0) 17:33:06 DEBUG --- stdout --- 17:33:06 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6176Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7490m 25% 6996Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 52m 0% 2683Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17585m 58% 12230Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 21330m 71% 16380Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5226Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17564m 58% 11859Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16902m 56% 3022Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19228m 64% 11213Mi 10% gke-xlou-cdm-ds-0f601751-j488 7581m 25% 6690Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21474m 71% 17496Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7489m 25% 7213Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21757m 72% 15698Mi 14% gke-xlou-cdm-frontend-a8771548-i82n 8172m 51% 5013Mi 8% 17:33:06 DEBUG --- stderr --- 17:33:06 DEBUG 17:34:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544443" 17:34:03 ERROR [http_cmd]: ERROR 17:34:03 ERROR --- status code --- 17:34:03 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:34:03 ERROR --- http response --- 17:34:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:34:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:34:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544443" 17:34:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544443" 17:34:03 INFO [http_cmd]: http status code OK 17:34:03 DEBUG --- status code --- 17:34:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:03 DEBUG --- http response --- 17:34:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:34:03 INFO [http_cmd]: http status code OK 17:34:03 DEBUG --- status code --- 17:34:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:03 DEBUG --- http response --- 17:34:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:34:03 DEBUG ----- output ----- 17:34:03 DEBUG { 17:34:03 DEBUG "status": "success", 17:34:03 DEBUG "data": { 17:34:03 DEBUG "resultType": "vector", 17:34:03 DEBUG "result": [ 17:34:03 DEBUG { 17:34:03 DEBUG "metric": { 17:34:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:34:03 DEBUG }, 17:34:03 DEBUG "value": [ 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG [...] 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:34:03 DEBUG }, 17:34:03 DEBUG "value": [ 17:34:03 DEBUG 1713544443, 17:34:03 DEBUG "2343.0666666666666" 17:34:03 DEBUG ] 17:34:03 DEBUG } 17:34:03 DEBUG ] 17:34:03 DEBUG } 17:34:03 DEBUG } 17:34:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544443" 17:34:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544443" 17:34:03 INFO [http_cmd]: http status code OK 17:34:03 INFO [http_cmd]: http status code OK 17:34:03 DEBUG --- status code --- 17:34:03 DEBUG --- status code --- 17:34:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:03 DEBUG --- http response --- 17:34:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:34:03 DEBUG ----- output ----- 17:34:03 DEBUG { 17:34:03 DEBUG "status": "success", 17:34:03 DEBUG "data": { 17:34:03 DEBUG "resultType": "vector", 17:34:03 DEBUG "result": [ 17:34:03 DEBUG { 17:34:03 DEBUG "metric": { 17:34:03 DEBUG "pod": "ds-idrepo-0" 17:34:03 DEBUG }, 17:34:03 DEBUG "value": [ 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG [...] 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG "pod": "ds-cts-2" 17:34:03 DEBUG }, 17:34:03 DEBUG "value": [ 17:34:03 DEBUG 1713544443, 17:34:03 DEBUG "0" 17:34:03 DEBUG ] 17:34:03 DEBUG } 17:34:03 DEBUG ] 17:34:03 DEBUG } 17:34:03 DEBUG } 17:34:03 DEBUG --- http response --- 17:34:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:34:03 DEBUG ----- output ----- 17:34:03 DEBUG { 17:34:03 DEBUG "status": "success", 17:34:03 DEBUG "data": { 17:34:03 DEBUG "resultType": "vector", 17:34:03 DEBUG "result": [ 17:34:03 DEBUG { 17:34:03 DEBUG "metric": { 17:34:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:34:03 DEBUG }, 17:34:03 DEBUG "value": [ 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG [...] 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:34:03 DEBUG }, 17:34:03 DEBUG "value": [ 17:34:03 DEBUG 1713544443, 17:34:03 DEBUG "2496.633333333333" 17:34:03 DEBUG ] 17:34:03 DEBUG } 17:34:03 DEBUG ] 17:34:03 DEBUG } 17:34:03 DEBUG } 17:34:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544443" 17:34:03 INFO [http_cmd]: http status code OK 17:34:03 DEBUG --- status code --- 17:34:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:03 DEBUG --- http response --- 17:34:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:34:03 DEBUG ----- output ----- 17:34:03 DEBUG { 17:34:03 DEBUG "status": "success", 17:34:03 DEBUG "data": { 17:34:03 DEBUG "resultType": "vector", 17:34:03 DEBUG "result": [ 17:34:03 DEBUG { 17:34:03 DEBUG "metric": { 17:34:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:34:03 DEBUG }, 17:34:03 DEBUG "value": [ 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG [...] 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG 17:34:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:34:03 DEBUG }, 17:34:03 DEBUG "value": [ 17:34:03 DEBUG 1713544443, 17:34:03 DEBUG "0" 17:34:03 DEBUG ] 17:34:03 DEBUG } 17:34:03 DEBUG ] 17:34:03 DEBUG } 17:34:03 DEBUG } 17:34:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544444" 17:34:04 INFO [http_cmd]: http status code OK 17:34:04 DEBUG --- status code --- 17:34:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:04 DEBUG --- http response --- 17:34:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:34:04 DEBUG ----- output ----- 17:34:04 DEBUG { 17:34:04 DEBUG "status": "success", 17:34:04 DEBUG "data": { 17:34:04 DEBUG "resultType": "vector", 17:34:04 DEBUG "result": [ 17:34:04 DEBUG { 17:34:04 DEBUG "metric": { 17:34:04 DEBUG "pod": "ds-idrepo-1" 17:34:04 DEBUG }, 17:34:04 DEBUG "value": [ 17:34:04 DEBUG 17:34:04 DEBUG 17:34:04 DEBUG 17:34:04 DEBUG [...] 17:34:04 DEBUG 17:34:04 DEBUG 17:34:04 DEBUG 17:34:04 DEBUG "pod": "ds-idrepo-2" 17:34:04 DEBUG }, 17:34:04 DEBUG "value": [ 17:34:04 DEBUG 1713544444, 17:34:04 DEBUG "1290.7027280334728" 17:34:04 DEBUG ] 17:34:04 DEBUG } 17:34:04 DEBUG ] 17:34:04 DEBUG } 17:34:04 DEBUG } 17:34:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544444" 17:34:04 INFO [http_cmd]: http status code OK 17:34:04 DEBUG --- status code --- 17:34:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:04 DEBUG --- http response --- 17:34:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:34:04 DEBUG ----- output ----- 17:34:04 DEBUG { 17:34:04 DEBUG "status": "success", 17:34:04 DEBUG "data": { 17:34:04 DEBUG "resultType": "vector", 17:34:04 DEBUG "result": [ 17:34:04 DEBUG { 17:34:04 DEBUG "metric": { 17:34:04 DEBUG "instance": "10.142.0.107:9100" 17:34:04 DEBUG }, 17:34:04 DEBUG "value": [ 17:34:04 DEBUG 17:34:04 DEBUG 17:34:04 DEBUG 17:34:04 DEBUG [...] 17:34:04 DEBUG 17:34:04 DEBUG 17:34:04 DEBUG 17:34:04 DEBUG "instance": "10.142.0.43:9100" 17:34:04 DEBUG }, 17:34:04 DEBUG "value": [ 17:34:04 DEBUG 1713544444, 17:34:04 DEBUG "0.37666666666665943" 17:34:04 DEBUG ] 17:34:04 DEBUG } 17:34:04 DEBUG ] 17:34:04 DEBUG } 17:34:04 DEBUG } 17:34:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544444" 17:34:04 INFO [http_cmd]: http status code OK 17:34:04 DEBUG --- status code --- 17:34:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:04 DEBUG --- http response --- 17:34:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:34:04 DEBUG ----- output ----- 17:34:04 DEBUG { 17:34:04 DEBUG "status": "success", 17:34:04 DEBUG "data": { 17:34:04 DEBUG "resultType": "vector", 17:34:04 DEBUG "result": [ 17:34:04 DEBUG { 17:34:04 DEBUG "metric": { 17:34:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:34:04 DEBUG }, 17:34:04 DEBUG "value": [ 17:34:04 DEBUG 17:34:04 DEBUG 17:34:04 DEBUG 17:34:04 DEBUG [...] 17:34:04 DEBUG 17:34:04 DEBUG 17:34:04 DEBUG 17:34:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:34:04 DEBUG }, 17:34:04 DEBUG "value": [ 17:34:04 DEBUG 1713544444, 17:34:04 DEBUG "4086783.5" 17:34:04 DEBUG ] 17:34:04 DEBUG } 17:34:04 DEBUG ] 17:34:04 DEBUG } 17:34:04 DEBUG } 17:34:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544445" 17:34:05 INFO [http_cmd]: http status code OK 17:34:05 DEBUG --- status code --- 17:34:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:05 DEBUG --- http response --- 17:34:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:34:05 DEBUG ----- output ----- 17:34:05 DEBUG { 17:34:05 DEBUG "status": "success", 17:34:05 DEBUG "data": { 17:34:05 DEBUG "resultType": "vector", 17:34:05 DEBUG "result": [ 17:34:05 DEBUG { 17:34:05 DEBUG "metric": { 17:34:05 DEBUG "instance": "10.142.0.107:9100" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG [...] 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG "instance": "10.142.0.43:9100" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 1713544445, 17:34:05 DEBUG "1719647304704" 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG } 17:34:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544445" 17:34:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544445" 17:34:05 INFO [http_cmd]: http status code OK 17:34:05 DEBUG --- status code --- 17:34:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:05 DEBUG --- http response --- 17:34:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:34:05 DEBUG ----- output ----- 17:34:05 DEBUG { 17:34:05 DEBUG "status": "success", 17:34:05 DEBUG "data": { 17:34:05 DEBUG "resultType": "vector", 17:34:05 DEBUG "result": [ 17:34:05 DEBUG { 17:34:05 DEBUG "metric": { 17:34:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG [...] 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 1713544445, 17:34:05 DEBUG "1414082560" 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG } 17:34:05 INFO [http_cmd]: http status code OK 17:34:05 DEBUG --- status code --- 17:34:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:05 DEBUG --- http response --- 17:34:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:34:05 DEBUG ----- output ----- 17:34:05 DEBUG { 17:34:05 DEBUG "status": "success", 17:34:05 DEBUG "data": { 17:34:05 DEBUG "resultType": "vector", 17:34:05 DEBUG "result": [ 17:34:05 DEBUG { 17:34:05 DEBUG "metric": { 17:34:05 DEBUG "pod": "ds-idrepo-0" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG [...] 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG "pod": "ds-idrepo-2" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 1713544445, 17:34:05 DEBUG "0" 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG } 17:34:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544445" 17:34:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544445" 17:34:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544445" 17:34:05 INFO [http_cmd]: http status code OK 17:34:05 DEBUG --- status code --- 17:34:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:05 DEBUG --- http response --- 17:34:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:34:05 DEBUG ----- output ----- 17:34:05 DEBUG { 17:34:05 DEBUG "status": "success", 17:34:05 DEBUG "data": { 17:34:05 DEBUG "resultType": "vector", 17:34:05 DEBUG "result": [ 17:34:05 DEBUG { 17:34:05 DEBUG "metric": { 17:34:05 DEBUG "pod": "ds-idrepo-0" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG [...] 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG "pod": "ds-idrepo-2" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 1713544445, 17:34:05 DEBUG "0" 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG } 17:34:05 INFO [http_cmd]: http status code OK 17:34:05 DEBUG --- status code --- 17:34:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:05 DEBUG --- http response --- 17:34:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:34:05 DEBUG ----- output ----- 17:34:05 DEBUG { 17:34:05 DEBUG "status": "success", 17:34:05 DEBUG "data": { 17:34:05 DEBUG "resultType": "vector", 17:34:05 DEBUG "result": [ 17:34:05 DEBUG { 17:34:05 DEBUG "metric": { 17:34:05 DEBUG "pod": "ds-cts-0" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG [...] 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG "pod": "ds-cts-2" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 1713544445, 17:34:05 DEBUG "0" 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG } 17:34:05 INFO [http_cmd]: http status code OK 17:34:05 DEBUG --- status code --- 17:34:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:05 DEBUG --- http response --- 17:34:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:34:05 DEBUG ----- output ----- 17:34:05 DEBUG { 17:34:05 DEBUG "status": "success", 17:34:05 DEBUG "data": { 17:34:05 DEBUG "resultType": "vector", 17:34:05 DEBUG "result": [ 17:34:05 DEBUG { 17:34:05 DEBUG "metric": { 17:34:05 DEBUG "pod": "ds-cts-0" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG [...] 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG "pod": "ds-cts-2" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 1713544445, 17:34:05 DEBUG "0.16523333333333334" 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG } 17:34:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544445" 17:34:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544445" 17:34:05 INFO [http_cmd]: http status code OK 17:34:05 DEBUG --- status code --- 17:34:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:05 DEBUG --- http response --- 17:34:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:34:05 DEBUG ----- output ----- 17:34:05 DEBUG { 17:34:05 DEBUG "status": "success", 17:34:05 DEBUG "data": { 17:34:05 DEBUG "resultType": "vector", 17:34:05 DEBUG "result": [ 17:34:05 DEBUG { 17:34:05 DEBUG "metric": { 17:34:05 DEBUG "pod": "ds-idrepo-1" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG [...] 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG "pod": "ds-idrepo-2" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 1713544445, 17:34:05 DEBUG "2198.746510460251" 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG } 17:34:05 INFO [http_cmd]: http status code OK 17:34:05 DEBUG --- status code --- 17:34:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:05 DEBUG --- http response --- 17:34:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:34:05 DEBUG ----- output ----- 17:34:05 DEBUG { 17:34:05 DEBUG "status": "success", 17:34:05 DEBUG "data": { 17:34:05 DEBUG "resultType": "vector", 17:34:05 DEBUG "result": [ 17:34:05 DEBUG { 17:34:05 DEBUG "metric": { 17:34:05 DEBUG "instance": "10.142.0.107:9100" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG [...] 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG "instance": "10.142.0.43:9100" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 1713544445, 17:34:05 DEBUG "75404048" 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG } 17:34:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544445" 17:34:05 INFO [http_cmd]: http status code OK 17:34:05 DEBUG --- status code --- 17:34:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:05 DEBUG --- http response --- 17:34:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:34:05 DEBUG ----- output ----- 17:34:05 DEBUG { 17:34:05 DEBUG "status": "success", 17:34:05 DEBUG "data": { 17:34:05 DEBUG "resultType": "vector", 17:34:05 DEBUG "result": [ 17:34:05 DEBUG { 17:34:05 DEBUG "metric": { 17:34:05 DEBUG "instance": "10.142.0.107:9100" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG [...] 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG 17:34:05 DEBUG "instance": "10.142.0.43:9100" 17:34:05 DEBUG }, 17:34:05 DEBUG "value": [ 17:34:05 DEBUG 1713544445, 17:34:05 DEBUG "75397750.36666667" 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG ] 17:34:05 DEBUG } 17:34:05 DEBUG } 17:34:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544446" 17:34:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544446" 17:34:06 INFO [http_cmd]: http status code OK 17:34:06 DEBUG --- status code --- 17:34:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:06 DEBUG --- http response --- 17:34:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:34:06 DEBUG ----- output ----- 17:34:06 DEBUG { 17:34:06 DEBUG "status": "success", 17:34:06 DEBUG "data": { 17:34:06 DEBUG "resultType": "vector", 17:34:06 DEBUG "result": [ 17:34:06 DEBUG { 17:34:06 DEBUG "metric": { 17:34:06 DEBUG "pod": "ds-idrepo-1" 17:34:06 DEBUG }, 17:34:06 DEBUG "value": [ 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG [...] 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:34:06 DEBUG }, 17:34:06 DEBUG "value": [ 17:34:06 DEBUG 1713544446, 17:34:06 DEBUG "0" 17:34:06 DEBUG ] 17:34:06 DEBUG } 17:34:06 DEBUG ] 17:34:06 DEBUG } 17:34:06 DEBUG } 17:34:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544446" 17:34:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544446" 17:34:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544446" 17:34:06 INFO [http_cmd]: http status code OK 17:34:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544446" 17:34:06 DEBUG --- status code --- 17:34:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544446" 17:34:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544446" 17:34:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544446" 17:34:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:06 DEBUG --- http response --- 17:34:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:34:06 INFO [http_cmd]: http status code OK 17:34:06 DEBUG ----- output ----- 17:34:06 DEBUG --- status code --- 17:34:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:06 DEBUG --- http response --- 17:34:06 DEBUG { 17:34:06 INFO [http_cmd]: http status code OK 17:34:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:34:06 INFO [http_cmd]: http status code OK 17:34:06 INFO [http_cmd]: http status code OK 17:34:06 DEBUG "status": "success", 17:34:06 DEBUG --- status code --- 17:34:06 INFO [http_cmd]: http status code OK 17:34:06 DEBUG ----- output ----- 17:34:06 DEBUG --- status code --- 17:34:06 DEBUG --- status code --- 17:34:06 INFO [http_cmd]: http status code OK 17:34:06 INFO [http_cmd]: http status code OK 17:34:06 DEBUG "data": { 17:34:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:06 DEBUG --- status code --- 17:34:06 DEBUG { 17:34:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:06 DEBUG --- http response --- 17:34:06 DEBUG --- status code --- 17:34:06 DEBUG "resultType": "vector", 17:34:06 DEBUG --- http response --- 17:34:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:06 DEBUG "status": "success", 17:34:06 DEBUG "data": { 17:34:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:34:06 DEBUG --- status code --- 17:34:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:06 DEBUG "result": [ 17:34:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:34:06 DEBUG --- http response --- 17:34:06 DEBUG "resultType": "vector", 17:34:06 DEBUG --- http response --- 17:34:06 DEBUG ----- output ----- 17:34:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:34:06 DEBUG --- http response --- 17:34:06 DEBUG { 17:34:06 DEBUG ----- output ----- 17:34:06 DEBUG "result": [ 17:34:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:34:06 DEBUG { 17:34:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:34:06 DEBUG --- http response --- 17:34:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:34:06 DEBUG "metric": { 17:34:06 DEBUG { 17:34:06 DEBUG { 17:34:06 DEBUG ----- output ----- 17:34:06 DEBUG "status": "success", 17:34:06 DEBUG ----- output ----- 17:34:06 DEBUG ----- output ----- 17:34:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:34:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:34:06 DEBUG "status": "success", 17:34:06 DEBUG "metric": { 17:34:06 DEBUG { 17:34:06 DEBUG "status": "success", 17:34:06 DEBUG { 17:34:06 DEBUG { 17:34:06 DEBUG "status": "success", 17:34:06 DEBUG }, 17:34:06 DEBUG "data": { 17:34:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:34:06 DEBUG "data": { 17:34:06 DEBUG "status": "success", 17:34:06 DEBUG "data": { 17:34:06 DEBUG ----- output ----- 17:34:06 DEBUG "value": [ 17:34:06 DEBUG "resultType": "vector", 17:34:06 DEBUG "data": { 17:34:06 DEBUG }, 17:34:06 DEBUG "value": [ 17:34:06 DEBUG "data": { 17:34:06 DEBUG "resultType": "vector", 17:34:06 DEBUG { 17:34:06 DEBUG 17:34:06 DEBUG "result": [ 17:34:06 DEBUG "resultType": "vector", 17:34:06 DEBUG 17:34:06 DEBUG "resultType": "vector", 17:34:06 DEBUG "result": [ 17:34:06 DEBUG "resultType": "vector", 17:34:06 DEBUG "status": "success", 17:34:06 DEBUG "data": { 17:34:06 DEBUG "resultType": "vector", 17:34:06 DEBUG "result": [ 17:34:06 DEBUG 17:34:06 DEBUG "result": [ 17:34:06 DEBUG { 17:34:06 DEBUG "result": [ 17:34:06 DEBUG "result": [ 17:34:06 DEBUG { 17:34:06 DEBUG 17:34:06 DEBUG { 17:34:06 DEBUG "metric": { 17:34:06 DEBUG { 17:34:06 DEBUG "metric": { 17:34:06 DEBUG { 17:34:06 DEBUG { 17:34:06 DEBUG "metric": { 17:34:06 DEBUG 17:34:06 DEBUG "pod": "ds-idrepo-1" 17:34:06 DEBUG 17:34:06 DEBUG "metric": { 17:34:06 DEBUG "pod": "ds-cts-0" 17:34:06 DEBUG }, 17:34:06 DEBUG "metric": { 17:34:06 DEBUG "pod": "ds-idrepo-0" 17:34:06 DEBUG [...] 17:34:06 DEBUG 17:34:06 DEBUG [...] 17:34:06 DEBUG "instance": "10.142.0.107:9100" 17:34:06 DEBUG "value": [ 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:34:06 DEBUG }, 17:34:06 DEBUG "value": [ 17:34:06 DEBUG 17:34:06 DEBUG }, 17:34:06 DEBUG 17:34:06 DEBUG "metric": { 17:34:06 DEBUG }, 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG }, 17:34:06 DEBUG 17:34:06 DEBUG "value": [ 17:34:06 DEBUG [...] 17:34:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:34:06 DEBUG }, 17:34:06 DEBUG "value": [ 17:34:06 DEBUG 17:34:06 DEBUG [...] 17:34:06 DEBUG 17:34:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:34:06 DEBUG }, 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG "value": [ 17:34:06 DEBUG 17:34:06 DEBUG "value": [ 17:34:06 DEBUG "value": [ 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG 1713544446, 17:34:06 DEBUG 17:34:06 DEBUG [...] 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG "pod": "ds-cts-2" 17:34:06 DEBUG }, 17:34:06 DEBUG "2493.633333333333" 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG "value": [ 17:34:06 DEBUG 1713544446, 17:34:06 DEBUG 17:34:06 DEBUG ] 17:34:06 DEBUG [...] 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:34:06 DEBUG "335.0333333333333" 17:34:06 DEBUG ] 17:34:06 DEBUG [...] 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:34:06 DEBUG }, 17:34:06 DEBUG "value": [ 17:34:06 DEBUG 1713544446, 17:34:06 DEBUG "8.020846326550739" 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG }, 17:34:06 DEBUG "value": [ 17:34:06 DEBUG } 17:34:06 DEBUG 17:34:06 DEBUG "pod": "ds-idrepo-2" 17:34:06 DEBUG ] 17:34:06 DEBUG 1713544446, 17:34:06 DEBUG 17:34:06 DEBUG } 17:34:06 DEBUG ] 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG }, 17:34:06 DEBUG } 17:34:06 DEBUG "0.023" 17:34:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:34:06 DEBUG ] 17:34:06 DEBUG } 17:34:06 DEBUG } 17:34:06 DEBUG "instance": "10.142.0.43:9100" 17:34:06 DEBUG "value": [ 17:34:06 DEBUG ] 17:34:06 DEBUG ] 17:34:06 DEBUG }, 17:34:06 DEBUG } 17:34:06 DEBUG 17:34:06 DEBUG }, 17:34:06 DEBUG 1713544446, 17:34:06 DEBUG } 17:34:06 DEBUG } 17:34:06 DEBUG ] 17:34:06 DEBUG } 17:34:06 DEBUG [...] 17:34:06 DEBUG "value": [ 17:34:06 DEBUG "0" 17:34:06 DEBUG } 17:34:06 DEBUG } 17:34:06 DEBUG 17:34:06 DEBUG 17:34:06 DEBUG 1713544446, 17:34:06 DEBUG ] 17:34:06 DEBUG } 17:34:06 DEBUG 17:34:06 DEBUG "2499.366666666667" 17:34:06 DEBUG "value": [ 17:34:06 DEBUG } 17:34:06 DEBUG "pod": "ds-idrepo-2" 17:34:06 DEBUG ] 17:34:06 DEBUG 1713544446, 17:34:06 DEBUG ] 17:34:06 DEBUG }, 17:34:06 DEBUG "value": [ 17:34:06 DEBUG 1713544446, 17:34:06 DEBUG } 17:34:06 DEBUG "0.8193999999999572" 17:34:06 DEBUG ] 17:34:06 DEBUG } 17:34:06 DEBUG } 17:34:06 DEBUG "0" 17:34:06 DEBUG } 17:34:06 DEBUG ] 17:34:06 DEBUG ] 17:34:06 DEBUG ] 17:34:06 DEBUG } 17:34:06 DEBUG } 17:34:06 DEBUG } 17:34:06 DEBUG } 17:34:06 DEBUG ] 17:34:06 DEBUG } 17:34:06 DEBUG } 17:34:06 DEBUG } 17:34:06 INFO 17:34:06 INFO [loop_until]: kubectl --namespace=xlou top pods 17:34:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:34:06 INFO [loop_until]: OK (rc = 0) 17:34:06 DEBUG --- stdout --- 17:34:06 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17095m 8961Mi am-6fd554f8f7-btd4s 16980m 8980Mi am-6fd554f8f7-wxcsj 18967m 8985Mi ds-cts-0 21583m 12986Mi ds-cts-1 21008m 12811Mi ds-cts-2 21658m 12931Mi ds-idrepo-0 7002m 3423Mi ds-idrepo-1 6817m 3387Mi ds-idrepo-2 7166m 3460Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7769m 1256Mi 17:34:06 DEBUG --- stderr --- 17:34:06 DEBUG 17:34:06 INFO 17:34:06 INFO [loop_until]: kubectl --namespace=xlou top node 17:34:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:34:06 INFO [loop_until]: OK (rc = 0) 17:34:06 DEBUG --- stdout --- 17:34:06 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6176Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7399m 24% 7002Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 51m 0% 2679Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17324m 58% 12253Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 21333m 71% 15955Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5209Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17232m 57% 11861Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16805m 56% 3010Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19180m 64% 11234Mi 10% gke-xlou-cdm-ds-0f601751-j488 7619m 25% 6695Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21414m 71% 17775Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7417m 24% 7212Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21648m 72% 15317Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8179m 51% 5018Mi 8% 17:34:06 DEBUG --- stderr --- 17:34:06 DEBUG 17:35:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544503" 17:35:03 ERROR [http_cmd]: ERROR 17:35:03 ERROR --- status code --- 17:35:03 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:35:03 ERROR --- http response --- 17:35:03 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:35:03 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:35:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544503" 17:35:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544503" 17:35:03 INFO [http_cmd]: http status code OK 17:35:03 DEBUG --- status code --- 17:35:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:03 DEBUG --- http response --- 17:35:03 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:35:03 INFO [http_cmd]: http status code OK 17:35:03 DEBUG --- status code --- 17:35:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:03 DEBUG --- http response --- 17:35:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:35:03 DEBUG ----- output ----- 17:35:03 DEBUG { 17:35:03 DEBUG "status": "success", 17:35:03 DEBUG "data": { 17:35:03 DEBUG "resultType": "vector", 17:35:03 DEBUG "result": [ 17:35:03 DEBUG { 17:35:03 DEBUG "metric": { 17:35:03 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:35:03 DEBUG }, 17:35:03 DEBUG "value": [ 17:35:03 DEBUG 17:35:03 DEBUG 17:35:03 DEBUG 17:35:03 DEBUG [...] 17:35:03 DEBUG 17:35:03 DEBUG 17:35:03 DEBUG 17:35:03 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:35:03 DEBUG }, 17:35:03 DEBUG "value": [ 17:35:03 DEBUG 1713544503, 17:35:03 DEBUG "2311.6666666666665" 17:35:03 DEBUG ] 17:35:03 DEBUG } 17:35:03 DEBUG ] 17:35:03 DEBUG } 17:35:03 DEBUG } 17:35:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544504" 17:35:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544504" 17:35:04 INFO [http_cmd]: http status code OK 17:35:04 INFO [http_cmd]: http status code OK 17:35:04 DEBUG --- status code --- 17:35:04 DEBUG --- status code --- 17:35:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:04 DEBUG --- http response --- 17:35:04 DEBUG --- http response --- 17:35:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:35:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:35:04 DEBUG ----- output ----- 17:35:04 DEBUG ----- output ----- 17:35:04 DEBUG { 17:35:04 DEBUG "status": "success", 17:35:04 DEBUG { 17:35:04 DEBUG "data": { 17:35:04 DEBUG "status": "success", 17:35:04 DEBUG "data": { 17:35:04 DEBUG "resultType": "vector", 17:35:04 DEBUG "result": [ 17:35:04 DEBUG "resultType": "vector", 17:35:04 DEBUG { 17:35:04 DEBUG "metric": { 17:35:04 DEBUG "result": [ 17:35:04 DEBUG { 17:35:04 DEBUG "metric": { 17:35:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:35:04 DEBUG "pod": "ds-idrepo-0" 17:35:04 DEBUG }, 17:35:04 DEBUG }, 17:35:04 DEBUG "value": [ 17:35:04 DEBUG "value": [ 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG [...] 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG "pod": "ds-cts-2" 17:35:04 DEBUG 17:35:04 DEBUG }, 17:35:04 DEBUG 17:35:04 DEBUG "value": [ 17:35:04 DEBUG [...] 17:35:04 DEBUG 1713544504, 17:35:04 DEBUG 17:35:04 DEBUG "0" 17:35:04 DEBUG ] 17:35:04 DEBUG 17:35:04 DEBUG } 17:35:04 DEBUG 17:35:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:35:04 DEBUG }, 17:35:04 DEBUG "value": [ 17:35:04 DEBUG 1713544504, 17:35:04 DEBUG ] 17:35:04 DEBUG "2504.233333333333" 17:35:04 DEBUG } 17:35:04 DEBUG ] 17:35:04 DEBUG } 17:35:04 DEBUG } 17:35:04 DEBUG ] 17:35:04 DEBUG } 17:35:04 DEBUG } 17:35:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544504" 17:35:04 INFO [http_cmd]: http status code OK 17:35:04 DEBUG --- status code --- 17:35:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:04 DEBUG --- http response --- 17:35:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:35:04 DEBUG ----- output ----- 17:35:04 DEBUG { 17:35:04 DEBUG "status": "success", 17:35:04 DEBUG "data": { 17:35:04 DEBUG "resultType": "vector", 17:35:04 DEBUG "result": [ 17:35:04 DEBUG { 17:35:04 DEBUG "metric": { 17:35:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:35:04 DEBUG }, 17:35:04 DEBUG "value": [ 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG [...] 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:35:04 DEBUG }, 17:35:04 DEBUG "value": [ 17:35:04 DEBUG 1713544504, 17:35:04 DEBUG "0" 17:35:04 DEBUG ] 17:35:04 DEBUG } 17:35:04 DEBUG ] 17:35:04 DEBUG } 17:35:04 DEBUG } 17:35:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544504" 17:35:04 INFO [http_cmd]: http status code OK 17:35:04 DEBUG --- status code --- 17:35:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:04 DEBUG --- http response --- 17:35:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:35:04 DEBUG ----- output ----- 17:35:04 DEBUG { 17:35:04 DEBUG "status": "success", 17:35:04 DEBUG "data": { 17:35:04 DEBUG "resultType": "vector", 17:35:04 DEBUG "result": [ 17:35:04 DEBUG { 17:35:04 DEBUG "metric": { 17:35:04 DEBUG "pod": "ds-idrepo-0" 17:35:04 DEBUG }, 17:35:04 DEBUG "value": [ 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG [...] 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:35:04 DEBUG }, 17:35:04 DEBUG "value": [ 17:35:04 DEBUG 1713544504, 17:35:04 DEBUG "0" 17:35:04 DEBUG ] 17:35:04 DEBUG } 17:35:04 DEBUG ] 17:35:04 DEBUG } 17:35:04 DEBUG } 17:35:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544504" 17:35:04 INFO [http_cmd]: http status code OK 17:35:04 DEBUG --- status code --- 17:35:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:04 DEBUG --- http response --- 17:35:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:35:04 DEBUG ----- output ----- 17:35:04 DEBUG { 17:35:04 DEBUG "status": "success", 17:35:04 DEBUG "data": { 17:35:04 DEBUG "resultType": "vector", 17:35:04 DEBUG "result": [ 17:35:04 DEBUG { 17:35:04 DEBUG "metric": { 17:35:04 DEBUG "instance": "10.142.0.107:9100" 17:35:04 DEBUG }, 17:35:04 DEBUG "value": [ 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG [...] 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG "instance": "10.142.0.43:9100" 17:35:04 DEBUG }, 17:35:04 DEBUG "value": [ 17:35:04 DEBUG 1713544504, 17:35:04 DEBUG "0.371999999999999" 17:35:04 DEBUG ] 17:35:04 DEBUG } 17:35:04 DEBUG ] 17:35:04 DEBUG } 17:35:04 DEBUG } 17:35:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544504" 17:35:04 INFO [http_cmd]: http status code OK 17:35:04 DEBUG --- status code --- 17:35:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:04 DEBUG --- http response --- 17:35:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:35:04 DEBUG ----- output ----- 17:35:04 DEBUG { 17:35:04 DEBUG "status": "success", 17:35:04 DEBUG "data": { 17:35:04 DEBUG "resultType": "vector", 17:35:04 DEBUG "result": [ 17:35:04 DEBUG { 17:35:04 DEBUG "metric": { 17:35:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:35:04 DEBUG }, 17:35:04 DEBUG "value": [ 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG [...] 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG 17:35:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:35:04 DEBUG }, 17:35:04 DEBUG "value": [ 17:35:04 DEBUG 1713544504, 17:35:04 DEBUG "4096739.5" 17:35:04 DEBUG ] 17:35:04 DEBUG } 17:35:04 DEBUG ] 17:35:04 DEBUG } 17:35:04 DEBUG } 17:35:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544505" 17:35:05 INFO [http_cmd]: http status code OK 17:35:05 DEBUG --- status code --- 17:35:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:05 DEBUG --- http response --- 17:35:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:35:05 DEBUG ----- output ----- 17:35:05 DEBUG { 17:35:05 DEBUG "status": "success", 17:35:05 DEBUG "data": { 17:35:05 DEBUG "resultType": "vector", 17:35:05 DEBUG "result": [ 17:35:05 DEBUG { 17:35:05 DEBUG "metric": { 17:35:05 DEBUG "instance": "10.142.0.107:9100" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG [...] 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG "instance": "10.142.0.43:9100" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 1713544505, 17:35:05 DEBUG "1723152020480" 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG } 17:35:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544505" 17:35:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544505" 17:35:05 INFO [http_cmd]: http status code OK 17:35:05 DEBUG --- status code --- 17:35:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:05 DEBUG --- http response --- 17:35:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:35:05 DEBUG ----- output ----- 17:35:05 DEBUG { 17:35:05 DEBUG "status": "success", 17:35:05 DEBUG "data": { 17:35:05 DEBUG "resultType": "vector", 17:35:05 DEBUG "result": [ 17:35:05 DEBUG { 17:35:05 DEBUG "metric": { 17:35:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG [...] 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 1713544505, 17:35:05 DEBUG "1416982528" 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG } 17:35:05 INFO [http_cmd]: http status code OK 17:35:05 DEBUG --- status code --- 17:35:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:05 DEBUG --- http response --- 17:35:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:35:05 DEBUG ----- output ----- 17:35:05 DEBUG { 17:35:05 DEBUG "status": "success", 17:35:05 DEBUG "data": { 17:35:05 DEBUG "resultType": "vector", 17:35:05 DEBUG "result": [ 17:35:05 DEBUG { 17:35:05 DEBUG "metric": { 17:35:05 DEBUG "pod": "ds-idrepo-0" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG [...] 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG "pod": "ds-idrepo-2" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 1713544505, 17:35:05 DEBUG "0" 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG } 17:35:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544505" 17:35:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544505" 17:35:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544505" 17:35:05 INFO [http_cmd]: http status code OK 17:35:05 DEBUG --- status code --- 17:35:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:05 DEBUG --- http response --- 17:35:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:35:05 DEBUG ----- output ----- 17:35:05 DEBUG { 17:35:05 DEBUG "status": "success", 17:35:05 DEBUG "data": { 17:35:05 DEBUG "resultType": "vector", 17:35:05 DEBUG "result": [ 17:35:05 DEBUG { 17:35:05 DEBUG "metric": { 17:35:05 DEBUG "pod": "ds-idrepo-0" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG [...] 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG "pod": "ds-idrepo-2" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 1713544505, 17:35:05 DEBUG "0" 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG } 17:35:05 INFO [http_cmd]: http status code OK 17:35:05 DEBUG --- status code --- 17:35:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:05 DEBUG --- http response --- 17:35:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:35:05 DEBUG ----- output ----- 17:35:05 DEBUG { 17:35:05 DEBUG "status": "success", 17:35:05 DEBUG "data": { 17:35:05 DEBUG "resultType": "vector", 17:35:05 DEBUG "result": [ 17:35:05 DEBUG { 17:35:05 DEBUG "metric": { 17:35:05 DEBUG "pod": "ds-cts-0" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG [...] 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG "pod": "ds-cts-2" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 1713544505, 17:35:05 DEBUG "0" 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG } 17:35:05 INFO [http_cmd]: http status code OK 17:35:05 DEBUG --- status code --- 17:35:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:05 DEBUG --- http response --- 17:35:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:35:05 DEBUG ----- output ----- 17:35:05 DEBUG { 17:35:05 DEBUG "status": "success", 17:35:05 DEBUG "data": { 17:35:05 DEBUG "resultType": "vector", 17:35:05 DEBUG "result": [ 17:35:05 DEBUG { 17:35:05 DEBUG "metric": { 17:35:05 DEBUG "pod": "ds-cts-0" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG [...] 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG "pod": "ds-cts-2" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 1713544505, 17:35:05 DEBUG "0.19150000000000003" 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG } 17:35:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544505" 17:35:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544505" 17:35:05 INFO [http_cmd]: http status code OK 17:35:05 DEBUG --- status code --- 17:35:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:05 DEBUG --- http response --- 17:35:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:35:05 DEBUG ----- output ----- 17:35:05 DEBUG { 17:35:05 DEBUG "status": "success", 17:35:05 DEBUG "data": { 17:35:05 DEBUG "resultType": "vector", 17:35:05 DEBUG "result": [ 17:35:05 DEBUG { 17:35:05 DEBUG "metric": { 17:35:05 DEBUG "pod": "ds-idrepo-0" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG [...] 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 1713544505, 17:35:05 DEBUG "0" 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG } 17:35:05 INFO [http_cmd]: http status code OK 17:35:05 DEBUG --- status code --- 17:35:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:05 DEBUG --- http response --- 17:35:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:35:05 DEBUG ----- output ----- 17:35:05 DEBUG { 17:35:05 DEBUG "status": "success", 17:35:05 DEBUG "data": { 17:35:05 DEBUG "resultType": "vector", 17:35:05 DEBUG "result": [ 17:35:05 DEBUG { 17:35:05 DEBUG "metric": { 17:35:05 DEBUG "instance": "10.142.0.107:9100" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG [...] 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG 17:35:05 DEBUG "instance": "10.142.0.43:9100" 17:35:05 DEBUG }, 17:35:05 DEBUG "value": [ 17:35:05 DEBUG 1713544505, 17:35:05 DEBUG "75409503.80000001" 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG ] 17:35:05 DEBUG } 17:35:05 DEBUG } 17:35:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544506" 17:35:06 INFO [http_cmd]: http status code OK 17:35:06 DEBUG --- status code --- 17:35:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:06 DEBUG --- http response --- 17:35:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:35:06 DEBUG ----- output ----- 17:35:06 DEBUG { 17:35:06 DEBUG "status": "success", 17:35:06 DEBUG "data": { 17:35:06 DEBUG "resultType": "vector", 17:35:06 DEBUG "result": [ 17:35:06 DEBUG { 17:35:06 DEBUG "metric": { 17:35:06 DEBUG "instance": "10.142.0.107:9100" 17:35:06 DEBUG }, 17:35:06 DEBUG "value": [ 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG [...] 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG "instance": "10.142.0.43:9100" 17:35:06 DEBUG }, 17:35:06 DEBUG "value": [ 17:35:06 DEBUG 1713544506, 17:35:06 DEBUG "75403747.83333334" 17:35:06 DEBUG ] 17:35:06 DEBUG } 17:35:06 DEBUG ] 17:35:06 DEBUG } 17:35:06 DEBUG } 17:35:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544506" 17:35:06 INFO [http_cmd]: http status code OK 17:35:06 DEBUG --- status code --- 17:35:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:06 DEBUG --- http response --- 17:35:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:35:06 DEBUG ----- output ----- 17:35:06 DEBUG { 17:35:06 DEBUG "status": "success", 17:35:06 DEBUG "data": { 17:35:06 DEBUG "resultType": "vector", 17:35:06 DEBUG "result": [ 17:35:06 DEBUG { 17:35:06 DEBUG "metric": { 17:35:06 DEBUG "pod": "ds-idrepo-0" 17:35:06 DEBUG }, 17:35:06 DEBUG "value": [ 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG [...] 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:35:06 DEBUG }, 17:35:06 DEBUG "value": [ 17:35:06 DEBUG 1713544506, 17:35:06 DEBUG "0" 17:35:06 DEBUG ] 17:35:06 DEBUG } 17:35:06 DEBUG ] 17:35:06 DEBUG } 17:35:06 DEBUG } 17:35:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544506" 17:35:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544506" 17:35:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544506" 17:35:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544506" 17:35:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544506" 17:35:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544506" 17:35:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544506" 17:35:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544506" 17:35:06 INFO [http_cmd]: http status code OK 17:35:06 INFO [http_cmd]: http status code OK 17:35:06 DEBUG --- status code --- 17:35:06 DEBUG --- status code --- 17:35:06 INFO [http_cmd]: http status code OK 17:35:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:06 INFO [http_cmd]: http status code OK 17:35:06 INFO [http_cmd]: http status code OK 17:35:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:06 DEBUG --- status code --- 17:35:06 DEBUG --- http response --- 17:35:06 DEBUG --- status code --- 17:35:06 DEBUG --- status code --- 17:35:06 DEBUG --- http response --- 17:35:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:06 INFO [http_cmd]: http status code OK 17:35:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:35:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:06 DEBUG --- http response --- 17:35:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:35:06 DEBUG --- status code --- 17:35:06 DEBUG ----- output ----- 17:35:06 DEBUG { 17:35:06 DEBUG --- http response --- 17:35:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:35:06 DEBUG ----- output ----- 17:35:06 DEBUG { 17:35:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:06 DEBUG --- http response --- 17:35:06 DEBUG "status": "success", 17:35:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:35:06 DEBUG ----- output ----- 17:35:06 INFO [http_cmd]: http status code OK 17:35:06 DEBUG "status": "success", 17:35:06 DEBUG --- http response --- 17:35:06 DEBUG "data": { 17:35:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:35:06 DEBUG ----- output ----- 17:35:06 DEBUG { 17:35:06 DEBUG "status": "success", 17:35:06 INFO [http_cmd]: http status code OK 17:35:06 DEBUG "data": { 17:35:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:35:06 DEBUG "resultType": "vector", 17:35:06 DEBUG ----- output ----- 17:35:06 DEBUG { 17:35:06 DEBUG "data": { 17:35:06 DEBUG --- status code --- 17:35:06 DEBUG --- status code --- 17:35:06 DEBUG "resultType": "vector", 17:35:06 DEBUG ----- output ----- 17:35:06 DEBUG "result": [ 17:35:06 DEBUG { 17:35:06 DEBUG "status": "success", 17:35:06 DEBUG "resultType": "vector", 17:35:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:35:06 DEBUG "result": [ 17:35:06 DEBUG { 17:35:06 DEBUG { 17:35:06 DEBUG "status": "success", 17:35:06 DEBUG "data": { 17:35:06 DEBUG "result": [ 17:35:06 DEBUG --- http response --- 17:35:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:35:06 DEBUG { 17:35:06 DEBUG "status": "success", 17:35:06 DEBUG "data": { 17:35:06 DEBUG "data": { 17:35:06 DEBUG "resultType": "vector", 17:35:06 DEBUG { 17:35:06 DEBUG --- http response --- 17:35:06 DEBUG ----- output ----- 17:35:06 DEBUG "metric": { 17:35:06 DEBUG "resultType": "vector", 17:35:06 DEBUG "resultType": "vector", 17:35:06 DEBUG "metric": { 17:35:06 DEBUG "result": [ 17:35:06 DEBUG "metric": { 17:35:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:35:06 DEBUG { 17:35:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:35:06 DEBUG "result": [ 17:35:06 DEBUG "result": [ 17:35:06 DEBUG "pod": "ds-cts-0" 17:35:06 DEBUG { 17:35:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:35:06 DEBUG ----- output ----- 17:35:06 DEBUG "status": "success", 17:35:06 DEBUG }, 17:35:06 DEBUG { 17:35:06 DEBUG { 17:35:06 DEBUG }, 17:35:06 DEBUG "metric": { 17:35:06 DEBUG }, 17:35:06 DEBUG { 17:35:06 DEBUG "data": { 17:35:06 DEBUG "value": [ 17:35:06 DEBUG "metric": { 17:35:06 DEBUG "metric": { 17:35:06 DEBUG "value": [ 17:35:06 DEBUG "pod": "ds-idrepo-0" 17:35:06 DEBUG "value": [ 17:35:06 DEBUG "status": "success", 17:35:06 DEBUG "data": { 17:35:06 DEBUG 17:35:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:35:06 DEBUG "pod": "ds-idrepo-0" 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG "resultType": "vector", 17:35:06 DEBUG "resultType": "vector", 17:35:06 DEBUG 17:35:06 DEBUG }, 17:35:06 DEBUG }, 17:35:06 DEBUG 17:35:06 DEBUG }, 17:35:06 DEBUG 17:35:06 DEBUG "result": [ 17:35:06 DEBUG "result": [ 17:35:06 DEBUG 17:35:06 DEBUG "value": [ 17:35:06 DEBUG "value": [ 17:35:06 DEBUG [...] 17:35:06 DEBUG "value": [ 17:35:06 DEBUG 17:35:06 DEBUG { 17:35:06 DEBUG { 17:35:06 DEBUG [...] 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG [...] 17:35:06 DEBUG "metric": { 17:35:06 DEBUG "metric": { 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:35:06 DEBUG 17:35:06 DEBUG "instance": "10.142.0.107:9100" 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG "pod": "ds-cts-2" 17:35:06 DEBUG 17:35:06 DEBUG }, 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG [...] 17:35:06 DEBUG [...] 17:35:06 DEBUG }, 17:35:06 DEBUG 17:35:06 DEBUG "value": [ 17:35:06 DEBUG [...] 17:35:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:35:06 DEBUG 17:35:06 DEBUG }, 17:35:06 DEBUG 17:35:06 DEBUG "value": [ 17:35:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG }, 17:35:06 DEBUG 17:35:06 DEBUG "value": [ 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG }, 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG "value": [ 17:35:06 DEBUG 17:35:06 DEBUG 1713544506, 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG "value": [ 17:35:06 DEBUG 17:35:06 DEBUG 17:35:06 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:35:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:35:06 DEBUG "305.93333333333334" 17:35:06 DEBUG ] 17:35:06 DEBUG 17:35:06 DEBUG [...] 17:35:06 DEBUG "pod": "ds-idrepo-2" 17:35:06 DEBUG }, 17:35:06 DEBUG "value": [ 17:35:06 DEBUG }, 17:35:06 DEBUG } 17:35:06 DEBUG ] 17:35:06 DEBUG 1713544506, 17:35:06 DEBUG "8.035025691525043" 17:35:06 DEBUG 1713544506, 17:35:06 DEBUG 1713544506, 17:35:06 DEBUG "value": [ 17:35:06 DEBUG [...] 17:35:06 DEBUG } 17:35:06 DEBUG }, 17:35:06 DEBUG ] 17:35:06 DEBUG 17:35:06 DEBUG "2505.9333333333334" 17:35:06 DEBUG "0.000005552054181184662" 17:35:06 DEBUG 1713544506, 17:35:06 DEBUG 17:35:06 DEBUG } 17:35:06 DEBUG "value": [ 17:35:06 DEBUG } 17:35:06 DEBUG 17:35:06 DEBUG ] 17:35:06 DEBUG ] 17:35:06 DEBUG "0.023" 17:35:06 DEBUG 17:35:06 DEBUG 1713544506, 17:35:06 DEBUG ] 17:35:06 DEBUG 17:35:06 DEBUG } 17:35:06 DEBUG } 17:35:06 DEBUG ] 17:35:06 DEBUG 17:35:06 DEBUG "0" 17:35:06 DEBUG } 17:35:06 DEBUG "instance": "10.142.0.43:9100" 17:35:06 DEBUG ] 17:35:06 DEBUG ] 17:35:06 DEBUG } 17:35:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:35:06 DEBUG ] 17:35:06 DEBUG } 17:35:06 DEBUG }, 17:35:06 DEBUG } 17:35:06 DEBUG } 17:35:06 DEBUG } 17:35:06 DEBUG ] 17:35:06 DEBUG } 17:35:06 DEBUG ] 17:35:06 DEBUG } 17:35:06 DEBUG }, 17:35:06 DEBUG } 17:35:06 DEBUG "value": [ 17:35:06 DEBUG } 17:35:06 DEBUG "value": [ 17:35:06 DEBUG } 17:35:06 DEBUG 1713544506, 17:35:06 DEBUG } 17:35:06 DEBUG 1713544506, 17:35:06 DEBUG "0.8102666666667299" 17:35:06 DEBUG "2503.866666666667" 17:35:06 DEBUG ] 17:35:06 DEBUG } 17:35:06 DEBUG ] 17:35:06 DEBUG } 17:35:06 DEBUG } 17:35:06 DEBUG ] 17:35:06 DEBUG } 17:35:06 DEBUG ] 17:35:06 DEBUG } 17:35:06 DEBUG } 17:35:06 INFO 17:35:06 INFO [loop_until]: kubectl --namespace=xlou top pods 17:35:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:35:06 INFO [loop_until]: OK (rc = 0) 17:35:06 DEBUG --- stdout --- 17:35:06 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17530m 8964Mi am-6fd554f8f7-btd4s 16973m 8986Mi am-6fd554f8f7-wxcsj 19020m 8989Mi ds-cts-0 22069m 13253Mi ds-cts-1 21438m 13060Mi ds-cts-2 21445m 12721Mi ds-idrepo-0 7016m 3422Mi ds-idrepo-1 6900m 3387Mi ds-idrepo-2 7013m 3461Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8051m 1260Mi 17:35:06 DEBUG --- stderr --- 17:35:06 DEBUG 17:35:06 INFO 17:35:06 INFO [loop_until]: kubectl --namespace=xlou top node 17:35:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:35:06 INFO [loop_until]: OK (rc = 0) 17:35:06 DEBUG --- stdout --- 17:35:06 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6180Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7458m 24% 7002Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 48m 0% 2679Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17215m 57% 12260Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 21395m 71% 16256Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5239Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17285m 57% 11873Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16860m 56% 3019Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19030m 63% 11224Mi 10% gke-xlou-cdm-ds-0f601751-j488 7569m 25% 6701Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 21460m 71% 17844Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7516m 25% 7212Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21739m 72% 15449Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8169m 51% 5013Mi 8% 17:35:06 DEBUG --- stderr --- 17:35:06 DEBUG 17:36:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544563" 17:36:04 ERROR [http_cmd]: ERROR 17:36:04 ERROR --- status code --- 17:36:04 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:36:04 ERROR --- http response --- 17:36:04 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:36:04 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:36:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544564" 17:36:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544564" 17:36:04 INFO [http_cmd]: http status code OK 17:36:04 DEBUG --- status code --- 17:36:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:04 DEBUG --- http response --- 17:36:04 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:36:04 INFO [http_cmd]: http status code OK 17:36:04 DEBUG --- status code --- 17:36:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:04 DEBUG --- http response --- 17:36:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:36:04 DEBUG ----- output ----- 17:36:04 DEBUG { 17:36:04 DEBUG "status": "success", 17:36:04 DEBUG "data": { 17:36:04 DEBUG "resultType": "vector", 17:36:04 DEBUG "result": [ 17:36:04 DEBUG { 17:36:04 DEBUG "metric": { 17:36:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG [...] 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 1713544564, 17:36:04 DEBUG "2338.7666666666664" 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG } 17:36:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544564" 17:36:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544564" 17:36:04 INFO [http_cmd]: http status code OK 17:36:04 INFO [http_cmd]: http status code OK 17:36:04 DEBUG --- status code --- 17:36:04 DEBUG --- status code --- 17:36:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:04 DEBUG --- http response --- 17:36:04 DEBUG --- http response --- 17:36:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:36:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:36:04 DEBUG ----- output ----- 17:36:04 DEBUG ----- output ----- 17:36:04 DEBUG { 17:36:04 DEBUG "status": "success", 17:36:04 DEBUG "data": { 17:36:04 DEBUG "resultType": "vector", 17:36:04 DEBUG "result": [ 17:36:04 DEBUG { 17:36:04 DEBUG { 17:36:04 DEBUG "status": "success", 17:36:04 DEBUG "metric": { 17:36:04 DEBUG "data": { 17:36:04 DEBUG "pod": "ds-idrepo-0" 17:36:04 DEBUG "resultType": "vector", 17:36:04 DEBUG }, 17:36:04 DEBUG "result": [ 17:36:04 DEBUG "value": [ 17:36:04 DEBUG { 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG [...] 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG "pod": "ds-cts-2" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 1713544564, 17:36:04 DEBUG "0" 17:36:04 DEBUG "metric": { 17:36:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG [...] 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 1713544564, 17:36:04 DEBUG "2497.1" 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG } 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG } 17:36:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544564" 17:36:04 INFO [http_cmd]: http status code OK 17:36:04 DEBUG --- status code --- 17:36:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:04 DEBUG --- http response --- 17:36:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:36:04 DEBUG ----- output ----- 17:36:04 DEBUG { 17:36:04 DEBUG "status": "success", 17:36:04 DEBUG "data": { 17:36:04 DEBUG "resultType": "vector", 17:36:04 DEBUG "result": [ 17:36:04 DEBUG { 17:36:04 DEBUG "metric": { 17:36:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG [...] 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 1713544564, 17:36:04 DEBUG "0" 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG } 17:36:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544564" 17:36:04 INFO [http_cmd]: http status code OK 17:36:04 DEBUG --- status code --- 17:36:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:04 DEBUG --- http response --- 17:36:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:36:04 DEBUG ----- output ----- 17:36:04 DEBUG { 17:36:04 DEBUG "status": "success", 17:36:04 DEBUG "data": { 17:36:04 DEBUG "resultType": "vector", 17:36:04 DEBUG "result": [ 17:36:04 DEBUG { 17:36:04 DEBUG "metric": { 17:36:04 DEBUG "pod": "ds-idrepo-1" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG [...] 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 1713544564, 17:36:04 DEBUG "0" 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG } 17:36:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544564" 17:36:04 INFO [http_cmd]: http status code OK 17:36:04 DEBUG --- status code --- 17:36:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:04 DEBUG --- http response --- 17:36:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:36:04 DEBUG ----- output ----- 17:36:04 DEBUG { 17:36:04 DEBUG "status": "success", 17:36:04 DEBUG "data": { 17:36:04 DEBUG "resultType": "vector", 17:36:04 DEBUG "result": [ 17:36:04 DEBUG { 17:36:04 DEBUG "metric": { 17:36:04 DEBUG "instance": "10.142.0.107:9100" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG [...] 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG "instance": "10.142.0.43:9100" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 1713544564, 17:36:04 DEBUG "0.38266666666666355" 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG } 17:36:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544564" 17:36:04 INFO [http_cmd]: http status code OK 17:36:04 DEBUG --- status code --- 17:36:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:04 DEBUG --- http response --- 17:36:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:36:04 DEBUG ----- output ----- 17:36:04 DEBUG { 17:36:04 DEBUG "status": "success", 17:36:04 DEBUG "data": { 17:36:04 DEBUG "resultType": "vector", 17:36:04 DEBUG "result": [ 17:36:04 DEBUG { 17:36:04 DEBUG "metric": { 17:36:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG [...] 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG 17:36:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:36:04 DEBUG }, 17:36:04 DEBUG "value": [ 17:36:04 DEBUG 1713544564, 17:36:04 DEBUG "4106868.5" 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG ] 17:36:04 DEBUG } 17:36:04 DEBUG } 17:36:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544565" 17:36:05 INFO [http_cmd]: http status code OK 17:36:05 DEBUG --- status code --- 17:36:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:05 DEBUG --- http response --- 17:36:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:36:05 DEBUG ----- output ----- 17:36:05 DEBUG { 17:36:05 DEBUG "status": "success", 17:36:05 DEBUG "data": { 17:36:05 DEBUG "resultType": "vector", 17:36:05 DEBUG "result": [ 17:36:05 DEBUG { 17:36:05 DEBUG "metric": { 17:36:05 DEBUG "instance": "10.142.0.107:9100" 17:36:05 DEBUG }, 17:36:05 DEBUG "value": [ 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG [...] 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG "instance": "10.142.0.43:9100" 17:36:05 DEBUG }, 17:36:05 DEBUG "value": [ 17:36:05 DEBUG 1713544565, 17:36:05 DEBUG "1726692606976" 17:36:05 DEBUG ] 17:36:05 DEBUG } 17:36:05 DEBUG ] 17:36:05 DEBUG } 17:36:05 DEBUG } 17:36:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544565" 17:36:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544565" 17:36:05 INFO [http_cmd]: http status code OK 17:36:05 DEBUG --- status code --- 17:36:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:05 DEBUG --- http response --- 17:36:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:36:05 DEBUG ----- output ----- 17:36:05 DEBUG { 17:36:05 DEBUG "status": "success", 17:36:05 DEBUG "data": { 17:36:05 DEBUG "resultType": "vector", 17:36:05 DEBUG "result": [ 17:36:05 DEBUG { 17:36:05 DEBUG "metric": { 17:36:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:36:05 DEBUG }, 17:36:05 DEBUG "value": [ 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG [...] 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 INFO [http_cmd]: http status code OK 17:36:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:36:05 DEBUG --- status code --- 17:36:05 DEBUG }, 17:36:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:05 DEBUG "value": [ 17:36:05 DEBUG --- http response --- 17:36:05 DEBUG 1713544565, 17:36:05 DEBUG "1420652544" 17:36:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:36:05 DEBUG ] 17:36:05 DEBUG ----- output ----- 17:36:05 DEBUG } 17:36:05 DEBUG { 17:36:05 DEBUG ] 17:36:05 DEBUG "status": "success", 17:36:05 DEBUG } 17:36:05 DEBUG "data": { 17:36:05 DEBUG } 17:36:05 DEBUG "resultType": "vector", 17:36:05 DEBUG "result": [ 17:36:05 DEBUG { 17:36:05 DEBUG "metric": { 17:36:05 DEBUG "pod": "ds-idrepo-0" 17:36:05 DEBUG }, 17:36:05 DEBUG "value": [ 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG [...] 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG "pod": "ds-idrepo-2" 17:36:05 DEBUG }, 17:36:05 DEBUG "value": [ 17:36:05 DEBUG 1713544565, 17:36:05 DEBUG "0" 17:36:05 DEBUG ] 17:36:05 DEBUG } 17:36:05 DEBUG ] 17:36:05 DEBUG } 17:36:05 DEBUG } 17:36:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544565" 17:36:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544565" 17:36:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544565" 17:36:05 INFO [http_cmd]: http status code OK 17:36:05 INFO [http_cmd]: http status code OK 17:36:05 DEBUG --- status code --- 17:36:05 DEBUG --- status code --- 17:36:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:05 DEBUG --- http response --- 17:36:05 DEBUG --- http response --- 17:36:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:36:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:36:05 DEBUG ----- output ----- 17:36:05 DEBUG ----- output ----- 17:36:05 DEBUG { 17:36:05 DEBUG { 17:36:05 DEBUG "status": "success", 17:36:05 DEBUG "status": "success", 17:36:05 DEBUG "data": { 17:36:05 DEBUG "data": { 17:36:05 DEBUG "resultType": "vector", 17:36:05 DEBUG "resultType": "vector", 17:36:05 DEBUG "result": [ 17:36:05 DEBUG "result": [ 17:36:05 DEBUG { 17:36:05 DEBUG { 17:36:05 DEBUG "metric": { 17:36:05 DEBUG "metric": { 17:36:05 DEBUG "pod": "ds-idrepo-0" 17:36:05 DEBUG "pod": "ds-cts-0" 17:36:05 DEBUG }, 17:36:05 DEBUG }, 17:36:05 DEBUG "value": [ 17:36:05 DEBUG "value": [ 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG [...] 17:36:05 DEBUG [...] 17:36:05 INFO [http_cmd]: http status code OK 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG --- status code --- 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG --- http response --- 17:36:05 DEBUG "pod": "ds-idrepo-2" 17:36:05 DEBUG "pod": "ds-cts-2" 17:36:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:36:05 DEBUG ----- output ----- 17:36:05 DEBUG }, 17:36:05 DEBUG }, 17:36:05 DEBUG { 17:36:05 DEBUG "value": [ 17:36:05 DEBUG "value": [ 17:36:05 DEBUG "status": "success", 17:36:05 DEBUG 1713544565, 17:36:05 DEBUG 1713544565, 17:36:05 DEBUG "data": { 17:36:05 DEBUG "0" 17:36:05 DEBUG "0.23403333333333337" 17:36:05 DEBUG "resultType": "vector", 17:36:05 DEBUG ] 17:36:05 DEBUG ] 17:36:05 DEBUG "result": [ 17:36:05 DEBUG } 17:36:05 DEBUG } 17:36:05 DEBUG { 17:36:05 DEBUG ] 17:36:05 DEBUG ] 17:36:05 DEBUG "metric": { 17:36:05 DEBUG } 17:36:05 DEBUG } 17:36:05 DEBUG "pod": "ds-cts-0" 17:36:05 DEBUG } 17:36:05 DEBUG } 17:36:05 DEBUG }, 17:36:05 DEBUG "value": [ 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG [...] 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG "pod": "ds-cts-2" 17:36:05 DEBUG }, 17:36:05 DEBUG "value": [ 17:36:05 DEBUG 1713544565, 17:36:05 DEBUG "0" 17:36:05 DEBUG ] 17:36:05 DEBUG } 17:36:05 DEBUG ] 17:36:05 DEBUG } 17:36:05 DEBUG } 17:36:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544565" 17:36:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544565" 17:36:05 INFO [http_cmd]: http status code OK 17:36:05 DEBUG --- status code --- 17:36:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:05 DEBUG --- http response --- 17:36:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:36:05 DEBUG ----- output ----- 17:36:05 DEBUG { 17:36:05 DEBUG "status": "success", 17:36:05 DEBUG "data": { 17:36:05 DEBUG "resultType": "vector", 17:36:05 DEBUG "result": [ 17:36:05 DEBUG { 17:36:05 DEBUG "metric": { 17:36:05 DEBUG "pod": "ds-idrepo-1" 17:36:05 DEBUG }, 17:36:05 DEBUG "value": [ 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG [...] 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:36:05 DEBUG }, 17:36:05 DEBUG "value": [ 17:36:05 DEBUG 1713544565, 17:36:05 DEBUG "0" 17:36:05 DEBUG ] 17:36:05 DEBUG } 17:36:05 DEBUG ] 17:36:05 DEBUG } 17:36:05 DEBUG } 17:36:05 INFO [http_cmd]: http status code OK 17:36:05 DEBUG --- status code --- 17:36:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:05 DEBUG --- http response --- 17:36:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:36:05 DEBUG ----- output ----- 17:36:05 DEBUG { 17:36:05 DEBUG "status": "success", 17:36:05 DEBUG "data": { 17:36:05 DEBUG "resultType": "vector", 17:36:05 DEBUG "result": [ 17:36:05 DEBUG { 17:36:05 DEBUG "metric": { 17:36:05 DEBUG "instance": "10.142.0.107:9100" 17:36:05 DEBUG }, 17:36:05 DEBUG "value": [ 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG [...] 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG 17:36:05 DEBUG "instance": "10.142.0.43:9100" 17:36:05 DEBUG }, 17:36:05 DEBUG "value": [ 17:36:05 DEBUG 1713544565, 17:36:05 DEBUG "75542702.80000001" 17:36:05 DEBUG ] 17:36:05 DEBUG } 17:36:05 DEBUG ] 17:36:05 DEBUG } 17:36:05 DEBUG } 17:36:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544566" 17:36:06 INFO [http_cmd]: http status code OK 17:36:06 DEBUG --- status code --- 17:36:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:06 DEBUG --- http response --- 17:36:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:36:06 DEBUG ----- output ----- 17:36:06 DEBUG { 17:36:06 DEBUG "status": "success", 17:36:06 DEBUG "data": { 17:36:06 DEBUG "resultType": "vector", 17:36:06 DEBUG "result": [ 17:36:06 DEBUG { 17:36:06 DEBUG "metric": { 17:36:06 DEBUG "instance": "10.142.0.107:9100" 17:36:06 DEBUG }, 17:36:06 DEBUG "value": [ 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG [...] 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG "instance": "10.142.0.43:9100" 17:36:06 DEBUG }, 17:36:06 DEBUG "value": [ 17:36:06 DEBUG 1713544566, 17:36:06 DEBUG "75536134.83333334" 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544566" 17:36:06 INFO [http_cmd]: http status code OK 17:36:06 DEBUG --- status code --- 17:36:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:06 DEBUG --- http response --- 17:36:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:36:06 DEBUG ----- output ----- 17:36:06 DEBUG { 17:36:06 DEBUG "status": "success", 17:36:06 DEBUG "data": { 17:36:06 DEBUG "resultType": "vector", 17:36:06 DEBUG "result": [ 17:36:06 DEBUG { 17:36:06 DEBUG "metric": { 17:36:06 DEBUG "pod": "ds-idrepo-1" 17:36:06 DEBUG }, 17:36:06 DEBUG "value": [ 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG [...] 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:36:06 DEBUG }, 17:36:06 DEBUG "value": [ 17:36:06 DEBUG 1713544566, 17:36:06 DEBUG "0" 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544566" 17:36:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544566" 17:36:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544566" 17:36:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544566" 17:36:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544566" 17:36:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544566" 17:36:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544566" 17:36:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544566" 17:36:06 INFO [http_cmd]: http status code OK 17:36:06 INFO [http_cmd]: http status code OK 17:36:06 INFO [http_cmd]: http status code OK 17:36:06 DEBUG --- status code --- 17:36:06 DEBUG --- status code --- 17:36:06 DEBUG --- status code --- 17:36:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:06 INFO [http_cmd]: http status code OK 17:36:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:06 INFO [http_cmd]: http status code OK 17:36:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:06 INFO [http_cmd]: http status code OK 17:36:06 DEBUG --- http response --- 17:36:06 DEBUG --- status code --- 17:36:06 DEBUG --- http response --- 17:36:06 DEBUG --- status code --- 17:36:06 DEBUG --- http response --- 17:36:06 DEBUG --- status code --- 17:36:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:36:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:06 INFO [http_cmd]: http status code OK 17:36:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:36:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:36:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:06 DEBUG ----- output ----- 17:36:06 DEBUG --- http response --- 17:36:06 INFO [http_cmd]: http status code OK 17:36:06 DEBUG --- status code --- 17:36:06 DEBUG --- http response --- 17:36:06 DEBUG ----- output ----- 17:36:06 DEBUG ----- output ----- 17:36:06 DEBUG --- http response --- 17:36:06 DEBUG { 17:36:06 DEBUG --- status code --- 17:36:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:36:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:36:06 DEBUG { 17:36:06 DEBUG { 17:36:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:36:06 DEBUG "status": "success", 17:36:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:36:06 DEBUG ----- output ----- 17:36:06 DEBUG --- http response --- 17:36:06 DEBUG ----- output ----- 17:36:06 DEBUG "status": "success", 17:36:06 DEBUG "status": "success", 17:36:06 DEBUG ----- output ----- 17:36:06 DEBUG "data": { 17:36:06 DEBUG --- http response --- 17:36:06 DEBUG { 17:36:06 DEBUG { 17:36:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:36:06 DEBUG "data": { 17:36:06 DEBUG "data": { 17:36:06 DEBUG { 17:36:06 DEBUG "resultType": "vector", 17:36:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:36:06 DEBUG "status": "success", 17:36:06 DEBUG "status": "success", 17:36:06 DEBUG ----- output ----- 17:36:06 DEBUG "resultType": "vector", 17:36:06 DEBUG "result": [ 17:36:06 DEBUG "status": "success", 17:36:06 DEBUG "result": [ 17:36:06 DEBUG ----- output ----- 17:36:06 DEBUG { 17:36:06 DEBUG "data": { 17:36:06 DEBUG { 17:36:06 DEBUG { 17:36:06 DEBUG "resultType": "vector", 17:36:06 DEBUG "data": { 17:36:06 DEBUG { 17:36:06 DEBUG "status": "success", 17:36:06 DEBUG "resultType": "vector", 17:36:06 DEBUG "status": "success", 17:36:06 DEBUG "metric": { 17:36:06 DEBUG "data": { 17:36:06 DEBUG "result": [ 17:36:06 DEBUG "resultType": "vector", 17:36:06 DEBUG "metric": { 17:36:06 DEBUG "pod": "ds-cts-0" 17:36:06 DEBUG }, 17:36:06 DEBUG "result": [ 17:36:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:36:06 DEBUG "resultType": "vector", 17:36:06 DEBUG { 17:36:06 DEBUG "result": [ 17:36:06 DEBUG "value": [ 17:36:06 DEBUG "data": { 17:36:06 DEBUG "data": { 17:36:06 DEBUG { 17:36:06 DEBUG }, 17:36:06 DEBUG "result": [ 17:36:06 DEBUG "metric": { 17:36:06 DEBUG { 17:36:06 DEBUG 17:36:06 DEBUG "resultType": "vector", 17:36:06 DEBUG "resultType": "vector", 17:36:06 DEBUG "metric": { 17:36:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:36:06 DEBUG { 17:36:06 DEBUG "pod": "ds-idrepo-1" 17:36:06 DEBUG "metric": { 17:36:06 DEBUG 17:36:06 DEBUG "result": [ 17:36:06 DEBUG "result": [ 17:36:06 DEBUG }, 17:36:06 DEBUG "value": [ 17:36:06 DEBUG "metric": { 17:36:06 DEBUG }, 17:36:06 DEBUG "pod": "ds-idrepo-0" 17:36:06 DEBUG 17:36:06 DEBUG { 17:36:06 DEBUG { 17:36:06 DEBUG "value": [ 17:36:06 DEBUG 17:36:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:36:06 DEBUG "value": [ 17:36:06 DEBUG }, 17:36:06 DEBUG [...] 17:36:06 DEBUG "metric": { 17:36:06 DEBUG "metric": { 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG }, 17:36:06 DEBUG 17:36:06 DEBUG "value": [ 17:36:06 DEBUG 17:36:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:36:06 DEBUG "instance": "10.142.0.107:9100" 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG "value": [ 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG }, 17:36:06 DEBUG "value": [ 17:36:06 DEBUG 17:36:06 DEBUG }, 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG [...] 17:36:06 DEBUG [...] 17:36:06 DEBUG "value": [ 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG "pod": "ds-cts-2" 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG [...] 17:36:06 DEBUG [...] 17:36:06 DEBUG }, 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG [...] 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG "value": [ 17:36:06 DEBUG [...] 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG [...] 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 1713544566, 17:36:06 DEBUG 17:36:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:36:06 DEBUG }, 17:36:06 DEBUG "value": [ 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG "315.1333333333333" 17:36:06 DEBUG 17:36:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:36:06 DEBUG 1713544566, 17:36:06 DEBUG 17:36:06 DEBUG 17:36:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:36:06 DEBUG "pod": "ds-idrepo-2" 17:36:06 DEBUG ] 17:36:06 DEBUG 17:36:06 DEBUG }, 17:36:06 DEBUG "8.040419944721513" 17:36:06 DEBUG 17:36:06 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:36:06 DEBUG }, 17:36:06 DEBUG }, 17:36:06 DEBUG } 17:36:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:36:06 DEBUG }, 17:36:06 DEBUG ] 17:36:06 DEBUG 17:36:06 DEBUG }, 17:36:06 DEBUG "value": [ 17:36:06 DEBUG "value": [ 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 DEBUG "value": [ 17:36:06 DEBUG "instance": "10.142.0.43:9100" 17:36:06 DEBUG "value": [ 17:36:06 DEBUG 1713544566, 17:36:06 DEBUG 1713544566, 17:36:06 DEBUG } 17:36:06 DEBUG "value": [ 17:36:06 DEBUG ] 17:36:06 DEBUG 1713544566, 17:36:06 DEBUG }, 17:36:06 DEBUG 1713544566, 17:36:06 DEBUG "2497.733333333333" 17:36:06 DEBUG "0" 17:36:06 DEBUG 1713544566, 17:36:06 DEBUG } 17:36:06 DEBUG "2497.0666666666666" 17:36:06 DEBUG "value": [ 17:36:06 DEBUG "0" 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG "0.023" 17:36:06 DEBUG } 17:36:06 DEBUG ] 17:36:06 DEBUG 1713544566, 17:36:06 DEBUG "0.8241666666666446" 17:36:06 DEBUG ] 17:36:06 DEBUG ] 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG ] 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG ] 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 DEBUG } 17:36:06 INFO 17:36:06 INFO [loop_until]: kubectl --namespace=xlou top pods 17:36:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:36:06 INFO [loop_until]: OK (rc = 0) 17:36:06 DEBUG --- stdout --- 17:36:06 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17527m 8968Mi am-6fd554f8f7-btd4s 17357m 9003Mi am-6fd554f8f7-wxcsj 19226m 8991Mi ds-cts-0 22028m 12875Mi ds-cts-1 21415m 12879Mi ds-cts-2 22650m 13047Mi ds-idrepo-0 6976m 3423Mi ds-idrepo-1 6955m 3388Mi ds-idrepo-2 7161m 3461Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7900m 1262Mi 17:36:06 DEBUG --- stderr --- 17:36:06 DEBUG 17:36:06 INFO 17:36:06 INFO [loop_until]: kubectl --namespace=xlou top node 17:36:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:36:07 INFO [loop_until]: OK (rc = 0) 17:36:07 DEBUG --- stdout --- 17:36:07 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6183Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7405m 24% 7008Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 48m 0% 2673Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17528m 58% 12265Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 21253m 71% 15929Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5229Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17567m 58% 11896Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16735m 56% 3017Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19218m 64% 11221Mi 10% gke-xlou-cdm-ds-0f601751-j488 7599m 25% 6697Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21438m 71% 17576Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7424m 24% 7209Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21798m 73% 15120Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7915m 49% 5017Mi 8% 17:36:07 DEBUG --- stderr --- 17:36:07 DEBUG 17:37:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544624" 17:37:04 ERROR [http_cmd]: ERROR 17:37:04 ERROR --- status code --- 17:37:04 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:37:04 ERROR --- http response --- 17:37:04 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:37:04 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:37:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544624" 17:37:04 INFO [http_cmd]: http status code OK 17:37:04 DEBUG --- status code --- 17:37:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:04 DEBUG --- http response --- 17:37:04 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:37:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544624" 17:37:04 INFO [http_cmd]: http status code OK 17:37:04 DEBUG --- status code --- 17:37:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:04 DEBUG --- http response --- 17:37:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:37:04 DEBUG ----- output ----- 17:37:04 DEBUG { 17:37:04 DEBUG "status": "success", 17:37:04 DEBUG "data": { 17:37:04 DEBUG "resultType": "vector", 17:37:04 DEBUG "result": [ 17:37:04 DEBUG { 17:37:04 DEBUG "metric": { 17:37:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG [...] 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 1713544624, 17:37:04 DEBUG "2327.8333333333335" 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG } 17:37:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544624" 17:37:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544624" 17:37:04 INFO [http_cmd]: http status code OK 17:37:04 DEBUG --- status code --- 17:37:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:04 DEBUG --- http response --- 17:37:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:37:04 DEBUG ----- output ----- 17:37:04 DEBUG { 17:37:04 DEBUG "status": "success", 17:37:04 DEBUG "data": { 17:37:04 DEBUG "resultType": "vector", 17:37:04 DEBUG "result": [ 17:37:04 DEBUG { 17:37:04 DEBUG "metric": { 17:37:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG [...] 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 1713544624, 17:37:04 DEBUG "2506.6666666666665" 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG } 17:37:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544624" 17:37:04 INFO [http_cmd]: http status code OK 17:37:04 DEBUG --- status code --- 17:37:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:04 DEBUG --- http response --- 17:37:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:37:04 DEBUG ----- output ----- 17:37:04 DEBUG { 17:37:04 DEBUG "status": "success", 17:37:04 DEBUG "data": { 17:37:04 DEBUG "resultType": "vector", 17:37:04 DEBUG "result": [ 17:37:04 DEBUG { 17:37:04 DEBUG "metric": { 17:37:04 DEBUG "pod": "ds-idrepo-0" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG [...] 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG "pod": "ds-cts-2" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 1713544624, 17:37:04 DEBUG "0" 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG } 17:37:04 INFO [http_cmd]: http status code OK 17:37:04 DEBUG --- status code --- 17:37:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:04 DEBUG --- http response --- 17:37:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:37:04 DEBUG ----- output ----- 17:37:04 DEBUG { 17:37:04 DEBUG "status": "success", 17:37:04 DEBUG "data": { 17:37:04 DEBUG "resultType": "vector", 17:37:04 DEBUG "result": [ 17:37:04 DEBUG { 17:37:04 DEBUG "metric": { 17:37:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG [...] 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 1713544624, 17:37:04 DEBUG "0" 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG } 17:37:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544624" 17:37:04 INFO [http_cmd]: http status code OK 17:37:04 DEBUG --- status code --- 17:37:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:04 DEBUG --- http response --- 17:37:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:37:04 DEBUG ----- output ----- 17:37:04 DEBUG { 17:37:04 DEBUG "status": "success", 17:37:04 DEBUG "data": { 17:37:04 DEBUG "resultType": "vector", 17:37:04 DEBUG "result": [ 17:37:04 DEBUG { 17:37:04 DEBUG "metric": { 17:37:04 DEBUG "pod": "ds-idrepo-1" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG [...] 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 1713544624, 17:37:04 DEBUG "0" 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG } 17:37:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544624" 17:37:04 INFO [http_cmd]: http status code OK 17:37:04 DEBUG --- status code --- 17:37:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:04 DEBUG --- http response --- 17:37:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:37:04 DEBUG ----- output ----- 17:37:04 DEBUG { 17:37:04 DEBUG "status": "success", 17:37:04 DEBUG "data": { 17:37:04 DEBUG "resultType": "vector", 17:37:04 DEBUG "result": [ 17:37:04 DEBUG { 17:37:04 DEBUG "metric": { 17:37:04 DEBUG "instance": "10.142.0.107:9100" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG [...] 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG "instance": "10.142.0.43:9100" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 1713544624, 17:37:04 DEBUG "0.38733333333332787" 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG } 17:37:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544624" 17:37:04 INFO [http_cmd]: http status code OK 17:37:04 DEBUG --- status code --- 17:37:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:04 DEBUG --- http response --- 17:37:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:37:04 DEBUG ----- output ----- 17:37:04 DEBUG { 17:37:04 DEBUG "status": "success", 17:37:04 DEBUG "data": { 17:37:04 DEBUG "resultType": "vector", 17:37:04 DEBUG "result": [ 17:37:04 DEBUG { 17:37:04 DEBUG "metric": { 17:37:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG [...] 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG 17:37:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:37:04 DEBUG }, 17:37:04 DEBUG "value": [ 17:37:04 DEBUG 1713544624, 17:37:04 DEBUG "4116689" 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG ] 17:37:04 DEBUG } 17:37:04 DEBUG } 17:37:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544625" 17:37:05 INFO [http_cmd]: http status code OK 17:37:05 DEBUG --- status code --- 17:37:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:05 DEBUG --- http response --- 17:37:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:37:05 DEBUG ----- output ----- 17:37:05 DEBUG { 17:37:05 DEBUG "status": "success", 17:37:05 DEBUG "data": { 17:37:05 DEBUG "resultType": "vector", 17:37:05 DEBUG "result": [ 17:37:05 DEBUG { 17:37:05 DEBUG "metric": { 17:37:05 DEBUG "instance": "10.142.0.107:9100" 17:37:05 DEBUG }, 17:37:05 DEBUG "value": [ 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG [...] 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG "instance": "10.142.0.43:9100" 17:37:05 DEBUG }, 17:37:05 DEBUG "value": [ 17:37:05 DEBUG 1713544625, 17:37:05 DEBUG "1730262227968" 17:37:05 DEBUG ] 17:37:05 DEBUG } 17:37:05 DEBUG ] 17:37:05 DEBUG } 17:37:05 DEBUG } 17:37:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544625" 17:37:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544625" 17:37:05 INFO [http_cmd]: http status code OK 17:37:05 DEBUG --- status code --- 17:37:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:05 DEBUG --- http response --- 17:37:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:37:05 DEBUG ----- output ----- 17:37:05 DEBUG { 17:37:05 DEBUG "status": "success", 17:37:05 DEBUG "data": { 17:37:05 DEBUG "resultType": "vector", 17:37:05 DEBUG "result": [ 17:37:05 DEBUG { 17:37:05 DEBUG "metric": { 17:37:05 DEBUG "pod": "ds-idrepo-0" 17:37:05 DEBUG }, 17:37:05 INFO [http_cmd]: http status code OK 17:37:05 DEBUG "value": [ 17:37:05 DEBUG --- status code --- 17:37:05 DEBUG 17:37:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:05 DEBUG 17:37:05 DEBUG --- http response --- 17:37:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:37:05 DEBUG ----- output ----- 17:37:05 DEBUG { 17:37:05 DEBUG "status": "success", 17:37:05 DEBUG "data": { 17:37:05 DEBUG "resultType": "vector", 17:37:05 DEBUG "result": [ 17:37:05 DEBUG { 17:37:05 DEBUG 17:37:05 DEBUG "metric": { 17:37:05 DEBUG [...] 17:37:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:37:05 DEBUG 17:37:05 DEBUG }, 17:37:05 DEBUG 17:37:05 DEBUG "value": [ 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG "pod": "ds-idrepo-2" 17:37:05 DEBUG [...] 17:37:05 DEBUG }, 17:37:05 DEBUG 17:37:05 DEBUG "value": [ 17:37:05 DEBUG 17:37:05 DEBUG 1713544625, 17:37:05 DEBUG "0" 17:37:05 DEBUG 17:37:05 DEBUG ] 17:37:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:37:05 DEBUG }, 17:37:05 DEBUG "value": [ 17:37:05 DEBUG 1713544625, 17:37:05 DEBUG "1425498112" 17:37:05 DEBUG ] 17:37:05 DEBUG } 17:37:05 DEBUG ] 17:37:05 DEBUG } 17:37:05 DEBUG } 17:37:05 DEBUG } 17:37:05 DEBUG ] 17:37:05 DEBUG } 17:37:05 DEBUG } 17:37:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544625" 17:37:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544625" 17:37:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544625" 17:37:05 INFO [http_cmd]: http status code OK 17:37:05 DEBUG --- status code --- 17:37:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:05 DEBUG --- http response --- 17:37:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:37:05 DEBUG ----- output ----- 17:37:05 DEBUG { 17:37:05 DEBUG "status": "success", 17:37:05 DEBUG "data": { 17:37:05 DEBUG "resultType": "vector", 17:37:05 DEBUG "result": [ 17:37:05 DEBUG { 17:37:05 DEBUG "metric": { 17:37:05 DEBUG "pod": "ds-cts-0" 17:37:05 DEBUG }, 17:37:05 DEBUG "value": [ 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG [...] 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG "pod": "ds-cts-2" 17:37:05 DEBUG }, 17:37:05 DEBUG "value": [ 17:37:05 DEBUG 1713544625, 17:37:05 DEBUG "0" 17:37:05 DEBUG ] 17:37:05 DEBUG } 17:37:05 DEBUG ] 17:37:05 DEBUG } 17:37:05 DEBUG } 17:37:05 INFO [http_cmd]: http status code OK 17:37:05 INFO [http_cmd]: http status code OK 17:37:05 DEBUG --- status code --- 17:37:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:05 DEBUG --- status code --- 17:37:05 DEBUG --- http response --- 17:37:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:37:05 DEBUG --- http response --- 17:37:05 DEBUG ----- output ----- 17:37:05 DEBUG { 17:37:05 DEBUG "status": "success", 17:37:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:37:05 DEBUG "data": { 17:37:05 DEBUG ----- output ----- 17:37:05 DEBUG "resultType": "vector", 17:37:05 DEBUG { 17:37:05 DEBUG "result": [ 17:37:05 DEBUG "status": "success", 17:37:05 DEBUG "data": { 17:37:05 DEBUG { 17:37:05 DEBUG "resultType": "vector", 17:37:05 DEBUG "metric": { 17:37:05 DEBUG "result": [ 17:37:05 DEBUG { 17:37:05 DEBUG "metric": { 17:37:05 DEBUG "pod": "ds-cts-0" 17:37:05 DEBUG }, 17:37:05 DEBUG "value": [ 17:37:05 DEBUG "pod": "ds-idrepo-0" 17:37:05 DEBUG 17:37:05 DEBUG }, 17:37:05 DEBUG 17:37:05 DEBUG "value": [ 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG [...] 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG [...] 17:37:05 DEBUG 17:37:05 DEBUG 17:37:05 DEBUG "pod": "ds-cts-2" 17:37:05 DEBUG 17:37:05 DEBUG }, 17:37:05 DEBUG 17:37:05 DEBUG "value": [ 17:37:05 DEBUG "pod": "ds-idrepo-2" 17:37:05 DEBUG 1713544625, 17:37:05 DEBUG }, 17:37:05 DEBUG "0.17739999999999997" 17:37:05 DEBUG "value": [ 17:37:05 DEBUG ] 17:37:05 DEBUG 1713544625, 17:37:05 DEBUG } 17:37:05 DEBUG "0" 17:37:05 DEBUG ] 17:37:05 DEBUG ] 17:37:05 DEBUG } 17:37:05 DEBUG } 17:37:05 DEBUG } 17:37:05 DEBUG ] 17:37:05 DEBUG } 17:37:05 DEBUG } 17:37:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544626" 17:37:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544626" 17:37:06 INFO [http_cmd]: http status code OK 17:37:06 DEBUG --- status code --- 17:37:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:06 DEBUG --- http response --- 17:37:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 17:37:06 DEBUG ----- output ----- 17:37:06 DEBUG { 17:37:06 DEBUG "status": "success", 17:37:06 DEBUG "data": { 17:37:06 DEBUG "resultType": "vector", 17:37:06 DEBUG "result": [ 17:37:06 DEBUG { 17:37:06 DEBUG "metric": { 17:37:06 DEBUG "pod": "ds-idrepo-0" 17:37:06 DEBUG }, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG [...] 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG "pod": "ds-idrepo-2" 17:37:06 DEBUG }, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 1713544626, 17:37:06 DEBUG "4377.33233204321" 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 INFO [http_cmd]: http status code OK 17:37:06 DEBUG --- status code --- 17:37:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:06 DEBUG --- http response --- 17:37:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:37:06 DEBUG ----- output ----- 17:37:06 DEBUG { 17:37:06 DEBUG "status": "success", 17:37:06 DEBUG "data": { 17:37:06 DEBUG "resultType": "vector", 17:37:06 DEBUG "result": [ 17:37:06 DEBUG { 17:37:06 DEBUG "metric": { 17:37:06 DEBUG "instance": "10.142.0.107:9100" 17:37:06 DEBUG }, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG [...] 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG "instance": "10.142.0.43:9100" 17:37:06 DEBUG }, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 1713544626, 17:37:06 DEBUG "75497926.26666668" 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544626" 17:37:06 INFO [http_cmd]: http status code OK 17:37:06 DEBUG --- status code --- 17:37:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:06 DEBUG --- http response --- 17:37:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:37:06 DEBUG ----- output ----- 17:37:06 DEBUG { 17:37:06 DEBUG "status": "success", 17:37:06 DEBUG "data": { 17:37:06 DEBUG "resultType": "vector", 17:37:06 DEBUG "result": [ 17:37:06 DEBUG { 17:37:06 DEBUG "metric": { 17:37:06 DEBUG "instance": "10.142.0.107:9100" 17:37:06 DEBUG }, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG [...] 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG "instance": "10.142.0.43:9100" 17:37:06 DEBUG }, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 1713544626, 17:37:06 DEBUG "75491168.2" 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544626" 17:37:06 INFO [http_cmd]: http status code OK 17:37:06 DEBUG --- status code --- 17:37:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:06 DEBUG --- http response --- 17:37:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 17:37:06 DEBUG ----- output ----- 17:37:06 DEBUG { 17:37:06 DEBUG "status": "success", 17:37:06 DEBUG "data": { 17:37:06 DEBUG "resultType": "vector", 17:37:06 DEBUG "result": [ 17:37:06 DEBUG { 17:37:06 DEBUG "metric": { 17:37:06 DEBUG "pod": "ds-idrepo-0" 17:37:06 DEBUG }, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG [...] 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG "pod": "ds-idrepo-2" 17:37:06 DEBUG }, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 1713544626, 17:37:06 DEBUG "0" 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544626" 17:37:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544626" 17:37:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544626" 17:37:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544626" 17:37:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544626" 17:37:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544626" 17:37:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544626" 17:37:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544626" 17:37:06 INFO [http_cmd]: http status code OK 17:37:06 INFO [http_cmd]: http status code OK 17:37:06 DEBUG --- status code --- 17:37:06 INFO [http_cmd]: http status code OK 17:37:06 DEBUG --- status code --- 17:37:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:06 INFO [http_cmd]: http status code OK 17:37:06 DEBUG --- status code --- 17:37:06 INFO [http_cmd]: http status code OK 17:37:06 INFO [http_cmd]: http status code OK 17:37:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:06 DEBUG --- http response --- 17:37:06 DEBUG --- http response --- 17:37:06 DEBUG --- status code --- 17:37:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:06 DEBUG --- status code --- 17:37:06 DEBUG --- status code --- 17:37:06 INFO [http_cmd]: http status code OK 17:37:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:37:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:37:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:06 DEBUG --- http response --- 17:37:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:06 DEBUG --- status code --- 17:37:06 DEBUG ----- output ----- 17:37:06 DEBUG ----- output ----- 17:37:06 INFO [http_cmd]: http status code OK 17:37:06 DEBUG --- http response --- 17:37:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:37:06 DEBUG --- http response --- 17:37:06 DEBUG --- http response --- 17:37:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:37:06 DEBUG ----- output ----- 17:37:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:06 DEBUG --- status code --- 17:37:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:37:06 DEBUG ----- output ----- 17:37:06 DEBUG { 17:37:06 DEBUG { 17:37:06 DEBUG { 17:37:06 DEBUG { 17:37:06 DEBUG --- http response --- 17:37:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:37:06 DEBUG ----- output ----- 17:37:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:37:06 DEBUG "status": "success", 17:37:06 DEBUG "status": "success", 17:37:06 DEBUG "data": { 17:37:06 DEBUG "status": "success", 17:37:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:37:06 DEBUG --- http response --- 17:37:06 DEBUG { 17:37:06 DEBUG ----- output ----- 17:37:06 DEBUG "data": { 17:37:06 DEBUG "resultType": "vector", 17:37:06 DEBUG "status": "success", 17:37:06 DEBUG "data": { 17:37:06 DEBUG ----- output ----- 17:37:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:37:06 DEBUG "status": "success", 17:37:06 DEBUG { 17:37:06 DEBUG "resultType": "vector", 17:37:06 DEBUG "result": [ 17:37:06 DEBUG "data": { 17:37:06 DEBUG "resultType": "vector", 17:37:06 DEBUG { 17:37:06 DEBUG ----- output ----- 17:37:06 DEBUG { 17:37:06 DEBUG "status": "success", 17:37:06 DEBUG "result": [ 17:37:06 DEBUG { 17:37:06 DEBUG "resultType": "vector", 17:37:06 DEBUG "result": [ 17:37:06 DEBUG "status": "success", 17:37:06 DEBUG "status": "success", 17:37:06 DEBUG "data": { 17:37:06 DEBUG "data": { 17:37:06 DEBUG { 17:37:06 DEBUG "metric": { 17:37:06 DEBUG "result": [ 17:37:06 DEBUG { 17:37:06 DEBUG "data": { 17:37:06 DEBUG "data": { 17:37:06 DEBUG "resultType": "vector", 17:37:06 DEBUG "resultType": "vector", 17:37:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:37:06 DEBUG }, 17:37:06 DEBUG { 17:37:06 DEBUG "metric": { 17:37:06 DEBUG "resultType": "vector", 17:37:06 DEBUG "resultType": "vector", 17:37:06 DEBUG "result": [ 17:37:06 DEBUG "result": [ 17:37:06 DEBUG "metric": { 17:37:06 DEBUG "value": [ 17:37:06 DEBUG "metric": { 17:37:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:37:06 DEBUG "result": [ 17:37:06 DEBUG { 17:37:06 DEBUG "result": [ 17:37:06 DEBUG { 17:37:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:37:06 DEBUG 17:37:06 DEBUG "pod": "ds-cts-0" 17:37:06 DEBUG }, 17:37:06 DEBUG { 17:37:06 DEBUG "metric": { 17:37:06 DEBUG "metric": { 17:37:06 DEBUG }, 17:37:06 DEBUG { 17:37:06 DEBUG "metric": { 17:37:06 DEBUG }, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG "metric": { 17:37:06 DEBUG "pod": "ds-idrepo-0" 17:37:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:37:06 DEBUG "value": [ 17:37:06 DEBUG "pod": "ds-idrepo-0" 17:37:06 DEBUG 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 17:37:06 DEBUG "instance": "10.142.0.107:9100" 17:37:06 DEBUG }, 17:37:06 DEBUG }, 17:37:06 DEBUG 17:37:06 DEBUG }, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG }, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG [...] 17:37:06 DEBUG 17:37:06 DEBUG [...] 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG [...] 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG [...] 17:37:06 DEBUG 17:37:06 DEBUG [...] 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG [...] 17:37:06 DEBUG [...] 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:37:06 DEBUG 17:37:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG }, 17:37:06 DEBUG 17:37:06 DEBUG }, 17:37:06 DEBUG [...] 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 17:37:06 DEBUG 17:37:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:37:06 DEBUG "pod": "ds-idrepo-2" 17:37:06 DEBUG 17:37:06 DEBUG 1713544626, 17:37:06 DEBUG "pod": "ds-cts-2" 17:37:06 DEBUG 1713544626, 17:37:06 DEBUG "0.023" 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG ] 17:37:06 DEBUG }, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG 1713544626, 17:37:06 DEBUG "7.127530154237458" 17:37:06 DEBUG }, 17:37:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG ] 17:37:06 DEBUG 17:37:06 DEBUG "value": [ 17:37:06 DEBUG }, 17:37:06 DEBUG } 17:37:06 DEBUG }, 17:37:06 DEBUG } 17:37:06 DEBUG "2504.3" 17:37:06 DEBUG "instance": "10.142.0.43:9100" 17:37:06 DEBUG }, 17:37:06 DEBUG 1713544626, 17:37:06 DEBUG "value": [ 17:37:06 DEBUG } 17:37:06 DEBUG "value": [ 17:37:06 DEBUG } 17:37:06 DEBUG ] 17:37:06 DEBUG "pod": "ds-idrepo-2" 17:37:06 DEBUG "value": [ 17:37:06 DEBUG "8.039257402477608" 17:37:06 DEBUG 1713544626, 17:37:06 DEBUG 1713544626, 17:37:06 DEBUG } 17:37:06 DEBUG }, 17:37:06 DEBUG 1713544626, 17:37:06 DEBUG "0.8250999999999455" 17:37:06 DEBUG ] 17:37:06 DEBUG "2505.8" 17:37:06 DEBUG ] 17:37:06 DEBUG "value": [ 17:37:06 DEBUG } 17:37:06 DEBUG "319.06666666666666" 17:37:06 DEBUG ] 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG 1713544626, 17:37:06 DEBUG "0" 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 DEBUG ] 17:37:06 DEBUG ] 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 DEBUG ] 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 DEBUG } 17:37:06 INFO 17:37:06 INFO [loop_until]: kubectl --namespace=xlou top pods 17:37:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:37:07 INFO [loop_until]: OK (rc = 0) 17:37:07 DEBUG --- stdout --- 17:37:07 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17248m 8971Mi am-6fd554f8f7-btd4s 17266m 9007Mi am-6fd554f8f7-wxcsj 18462m 8998Mi ds-cts-0 21338m 13058Mi ds-cts-1 21736m 12860Mi ds-cts-2 21677m 12753Mi ds-idrepo-0 6888m 3423Mi ds-idrepo-1 6964m 3391Mi ds-idrepo-2 7076m 3461Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8023m 1263Mi 17:37:07 DEBUG --- stderr --- 17:37:07 DEBUG 17:37:07 INFO 17:37:07 INFO [loop_until]: kubectl --namespace=xlou top node 17:37:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:37:07 INFO [loop_until]: OK (rc = 0) 17:37:07 DEBUG --- stdout --- 17:37:07 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6182Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7387m 24% 6996Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 45m 0% 2685Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17571m 58% 12262Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 21350m 71% 16176Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5240Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17610m 58% 11897Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16651m 55% 3016Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19142m 64% 11229Mi 10% gke-xlou-cdm-ds-0f601751-j488 7609m 25% 6692Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21440m 71% 17574Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7428m 24% 7206Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21799m 73% 15259Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8177m 51% 5018Mi 8% 17:37:07 DEBUG --- stderr --- 17:37:07 DEBUG 17:38:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544684" 17:38:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544684" 17:38:04 ERROR [http_cmd]: ERROR 17:38:04 ERROR --- status code --- 17:38:04 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:38:04 ERROR --- http response --- 17:38:04 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:38:04 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:38:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544684" 17:38:04 INFO [http_cmd]: http status code OK 17:38:04 DEBUG --- status code --- 17:38:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:04 DEBUG --- http response --- 17:38:04 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:38:04 INFO [http_cmd]: http status code OK 17:38:04 DEBUG --- status code --- 17:38:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:04 DEBUG --- http response --- 17:38:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:38:04 DEBUG ----- output ----- 17:38:04 DEBUG { 17:38:04 DEBUG "status": "success", 17:38:04 DEBUG "data": { 17:38:04 DEBUG "resultType": "vector", 17:38:04 DEBUG "result": [ 17:38:04 DEBUG { 17:38:04 DEBUG "metric": { 17:38:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:38:04 DEBUG }, 17:38:04 DEBUG "value": [ 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG [...] 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:38:04 DEBUG }, 17:38:04 DEBUG "value": [ 17:38:04 DEBUG 1713544684, 17:38:04 DEBUG "2332.9" 17:38:04 DEBUG ] 17:38:04 DEBUG } 17:38:04 DEBUG ] 17:38:04 DEBUG } 17:38:04 DEBUG } 17:38:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544684" 17:38:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544684" 17:38:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544684" 17:38:04 INFO [http_cmd]: http status code OK 17:38:04 DEBUG --- status code --- 17:38:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:04 DEBUG --- http response --- 17:38:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:38:04 DEBUG ----- output ----- 17:38:04 DEBUG { 17:38:04 DEBUG "status": "success", 17:38:04 DEBUG "data": { 17:38:04 DEBUG "resultType": "vector", 17:38:04 DEBUG "result": [ 17:38:04 DEBUG { 17:38:04 DEBUG "metric": { 17:38:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:38:04 DEBUG }, 17:38:04 DEBUG "value": [ 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG [...] 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:38:04 DEBUG }, 17:38:04 DEBUG "value": [ 17:38:04 DEBUG 1713544684, 17:38:04 DEBUG "2505.366666666667" 17:38:04 INFO [http_cmd]: http status code OK 17:38:04 DEBUG ] 17:38:04 INFO [http_cmd]: http status code OK 17:38:04 DEBUG --- status code --- 17:38:04 DEBUG } 17:38:04 DEBUG --- status code --- 17:38:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:04 DEBUG ] 17:38:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:04 DEBUG --- http response --- 17:38:04 DEBUG } 17:38:04 DEBUG } 17:38:04 DEBUG --- http response --- 17:38:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:38:04 DEBUG ----- output ----- 17:38:04 DEBUG { 17:38:04 DEBUG "status": "success", 17:38:04 DEBUG "data": { 17:38:04 DEBUG "resultType": "vector", 17:38:04 DEBUG "result": [ 17:38:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:38:04 DEBUG { 17:38:04 DEBUG ----- output ----- 17:38:04 DEBUG "metric": { 17:38:04 DEBUG { 17:38:04 DEBUG "status": "success", 17:38:04 DEBUG "data": { 17:38:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:38:04 DEBUG "resultType": "vector", 17:38:04 DEBUG }, 17:38:04 DEBUG "result": [ 17:38:04 DEBUG "value": [ 17:38:04 DEBUG { 17:38:04 DEBUG 17:38:04 DEBUG "metric": { 17:38:04 DEBUG 17:38:04 DEBUG "pod": "ds-idrepo-0" 17:38:04 DEBUG 17:38:04 DEBUG }, 17:38:04 DEBUG [...] 17:38:04 DEBUG "value": [ 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:38:04 DEBUG [...] 17:38:04 DEBUG }, 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG "pod": "ds-cts-2" 17:38:04 DEBUG }, 17:38:04 DEBUG "value": [ 17:38:04 DEBUG 1713544684, 17:38:04 DEBUG "value": [ 17:38:04 DEBUG "0" 17:38:04 DEBUG 1713544684, 17:38:04 DEBUG ] 17:38:04 DEBUG "0" 17:38:04 DEBUG } 17:38:04 DEBUG ] 17:38:04 DEBUG ] 17:38:04 DEBUG } 17:38:04 DEBUG } 17:38:04 DEBUG } 17:38:04 DEBUG ] 17:38:04 DEBUG } 17:38:04 DEBUG } 17:38:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544684" 17:38:04 INFO [http_cmd]: http status code OK 17:38:04 DEBUG --- status code --- 17:38:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:04 DEBUG --- http response --- 17:38:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:38:04 DEBUG ----- output ----- 17:38:04 DEBUG { 17:38:04 DEBUG "status": "success", 17:38:04 DEBUG "data": { 17:38:04 DEBUG "resultType": "vector", 17:38:04 DEBUG "result": [ 17:38:04 DEBUG { 17:38:04 DEBUG "metric": { 17:38:04 DEBUG "pod": "ds-idrepo-0" 17:38:04 DEBUG }, 17:38:04 DEBUG "value": [ 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG [...] 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:38:04 DEBUG }, 17:38:04 DEBUG "value": [ 17:38:04 DEBUG 1713544684, 17:38:04 DEBUG "0" 17:38:04 DEBUG ] 17:38:04 DEBUG } 17:38:04 DEBUG ] 17:38:04 DEBUG } 17:38:04 DEBUG } 17:38:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544684" 17:38:04 INFO [http_cmd]: http status code OK 17:38:04 DEBUG --- status code --- 17:38:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:04 DEBUG --- http response --- 17:38:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:38:04 DEBUG ----- output ----- 17:38:04 DEBUG { 17:38:04 DEBUG "status": "success", 17:38:04 DEBUG "data": { 17:38:04 DEBUG "resultType": "vector", 17:38:04 DEBUG "result": [ 17:38:04 DEBUG { 17:38:04 DEBUG "metric": { 17:38:04 DEBUG "instance": "10.142.0.107:9100" 17:38:04 DEBUG }, 17:38:04 DEBUG "value": [ 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG [...] 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG "instance": "10.142.0.43:9100" 17:38:04 DEBUG }, 17:38:04 DEBUG "value": [ 17:38:04 DEBUG 1713544684, 17:38:04 DEBUG "0.44633333333334047" 17:38:04 DEBUG ] 17:38:04 DEBUG } 17:38:04 DEBUG ] 17:38:04 DEBUG } 17:38:04 DEBUG } 17:38:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544684" 17:38:04 INFO [http_cmd]: http status code OK 17:38:04 DEBUG --- status code --- 17:38:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:04 DEBUG --- http response --- 17:38:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:38:04 DEBUG ----- output ----- 17:38:04 DEBUG { 17:38:04 DEBUG "status": "success", 17:38:04 DEBUG "data": { 17:38:04 DEBUG "resultType": "vector", 17:38:04 DEBUG "result": [ 17:38:04 DEBUG { 17:38:04 DEBUG "metric": { 17:38:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:38:04 DEBUG }, 17:38:04 DEBUG "value": [ 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG [...] 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG 17:38:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:38:04 DEBUG }, 17:38:04 DEBUG "value": [ 17:38:04 DEBUG 1713544684, 17:38:04 DEBUG "4126855.5" 17:38:04 DEBUG ] 17:38:04 DEBUG } 17:38:04 DEBUG ] 17:38:04 DEBUG } 17:38:04 DEBUG } 17:38:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544685" 17:38:05 INFO [http_cmd]: http status code OK 17:38:05 DEBUG --- status code --- 17:38:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:05 DEBUG --- http response --- 17:38:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:38:05 DEBUG ----- output ----- 17:38:05 DEBUG { 17:38:05 DEBUG "status": "success", 17:38:05 DEBUG "data": { 17:38:05 DEBUG "resultType": "vector", 17:38:05 DEBUG "result": [ 17:38:05 DEBUG { 17:38:05 DEBUG "metric": { 17:38:05 DEBUG "instance": "10.142.0.107:9100" 17:38:05 DEBUG }, 17:38:05 DEBUG "value": [ 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG [...] 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG "instance": "10.142.0.43:9100" 17:38:05 DEBUG }, 17:38:05 DEBUG "value": [ 17:38:05 DEBUG 1713544685, 17:38:05 DEBUG "1733805919232" 17:38:05 DEBUG ] 17:38:05 DEBUG } 17:38:05 DEBUG ] 17:38:05 DEBUG } 17:38:05 DEBUG } 17:38:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544685" 17:38:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544685" 17:38:05 INFO [http_cmd]: http status code OK 17:38:05 DEBUG --- status code --- 17:38:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:05 DEBUG --- http response --- 17:38:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:38:05 DEBUG ----- output ----- 17:38:05 DEBUG { 17:38:05 DEBUG "status": "success", 17:38:05 DEBUG "data": { 17:38:05 DEBUG "resultType": "vector", 17:38:05 DEBUG "result": [ 17:38:05 DEBUG { 17:38:05 DEBUG "metric": { 17:38:05 DEBUG "pod": "ds-idrepo-0" 17:38:05 DEBUG }, 17:38:05 DEBUG "value": [ 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG [...] 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG "pod": "ds-idrepo-2" 17:38:05 DEBUG }, 17:38:05 DEBUG "value": [ 17:38:05 DEBUG 1713544685, 17:38:05 DEBUG "0" 17:38:05 DEBUG ] 17:38:05 DEBUG } 17:38:05 DEBUG ] 17:38:05 DEBUG } 17:38:05 DEBUG } 17:38:05 INFO [http_cmd]: http status code OK 17:38:05 DEBUG --- status code --- 17:38:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:05 DEBUG --- http response --- 17:38:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:38:05 DEBUG ----- output ----- 17:38:05 DEBUG { 17:38:05 DEBUG "status": "success", 17:38:05 DEBUG "data": { 17:38:05 DEBUG "resultType": "vector", 17:38:05 DEBUG "result": [ 17:38:05 DEBUG { 17:38:05 DEBUG "metric": { 17:38:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:38:05 DEBUG }, 17:38:05 DEBUG "value": [ 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG [...] 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:38:05 DEBUG }, 17:38:05 DEBUG "value": [ 17:38:05 DEBUG 1713544685, 17:38:05 DEBUG "1423904768" 17:38:05 DEBUG ] 17:38:05 DEBUG } 17:38:05 DEBUG ] 17:38:05 DEBUG } 17:38:05 DEBUG } 17:38:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544685" 17:38:05 INFO [http_cmd]: http status code OK 17:38:05 DEBUG --- status code --- 17:38:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544685" 17:38:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544685" 17:38:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:05 DEBUG --- http response --- 17:38:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:38:05 DEBUG ----- output ----- 17:38:05 DEBUG { 17:38:05 DEBUG "status": "success", 17:38:05 DEBUG "data": { 17:38:05 DEBUG "resultType": "vector", 17:38:05 DEBUG "result": [ 17:38:05 DEBUG { 17:38:05 DEBUG "metric": { 17:38:05 DEBUG "pod": "ds-cts-0" 17:38:05 DEBUG }, 17:38:05 DEBUG "value": [ 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG [...] 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG "pod": "ds-cts-2" 17:38:05 DEBUG }, 17:38:05 DEBUG "value": [ 17:38:05 DEBUG 1713544685, 17:38:05 DEBUG "0" 17:38:05 DEBUG ] 17:38:05 DEBUG } 17:38:05 DEBUG ] 17:38:05 DEBUG } 17:38:05 DEBUG } 17:38:05 INFO [http_cmd]: http status code OK 17:38:05 DEBUG --- status code --- 17:38:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:05 DEBUG --- http response --- 17:38:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:38:05 DEBUG ----- output ----- 17:38:05 DEBUG { 17:38:05 DEBUG "status": "success", 17:38:05 DEBUG "data": { 17:38:05 DEBUG "resultType": "vector", 17:38:05 DEBUG "result": [ 17:38:05 DEBUG { 17:38:05 DEBUG "metric": { 17:38:05 DEBUG "pod": "ds-idrepo-0" 17:38:05 DEBUG }, 17:38:05 DEBUG "value": [ 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG [...] 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG "pod": "ds-idrepo-2" 17:38:05 DEBUG }, 17:38:05 DEBUG "value": [ 17:38:05 DEBUG 1713544685, 17:38:05 DEBUG "0" 17:38:05 DEBUG ] 17:38:05 DEBUG } 17:38:05 INFO [http_cmd]: http status code OK 17:38:05 DEBUG ] 17:38:05 DEBUG --- status code --- 17:38:05 DEBUG } 17:38:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:05 DEBUG } 17:38:05 DEBUG --- http response --- 17:38:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:38:05 DEBUG ----- output ----- 17:38:05 DEBUG { 17:38:05 DEBUG "status": "success", 17:38:05 DEBUG "data": { 17:38:05 DEBUG "resultType": "vector", 17:38:05 DEBUG "result": [ 17:38:05 DEBUG { 17:38:05 DEBUG "metric": { 17:38:05 DEBUG "pod": "ds-cts-0" 17:38:05 DEBUG }, 17:38:05 DEBUG "value": [ 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG [...] 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG 17:38:05 DEBUG "pod": "ds-cts-2" 17:38:05 DEBUG }, 17:38:05 DEBUG "value": [ 17:38:05 DEBUG 1713544685, 17:38:05 DEBUG "0.20086666666666675" 17:38:05 DEBUG ] 17:38:05 DEBUG } 17:38:05 DEBUG ] 17:38:05 DEBUG } 17:38:05 DEBUG } 17:38:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544686" 17:38:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544686" 17:38:06 INFO [http_cmd]: http status code OK 17:38:06 INFO [http_cmd]: http status code OK 17:38:06 DEBUG --- status code --- 17:38:06 DEBUG --- status code --- 17:38:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:06 DEBUG --- http response --- 17:38:06 DEBUG --- http response --- 17:38:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:38:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:38:06 DEBUG ----- output ----- 17:38:06 DEBUG ----- output ----- 17:38:06 DEBUG { 17:38:06 DEBUG { 17:38:06 DEBUG "status": "success", 17:38:06 DEBUG "status": "success", 17:38:06 DEBUG "data": { 17:38:06 DEBUG "data": { 17:38:06 DEBUG "resultType": "vector", 17:38:06 DEBUG "resultType": "vector", 17:38:06 DEBUG "result": [ 17:38:06 DEBUG "result": [ 17:38:06 DEBUG { 17:38:06 DEBUG { 17:38:06 DEBUG "metric": { 17:38:06 DEBUG "metric": { 17:38:06 DEBUG "pod": "ds-idrepo-0" 17:38:06 DEBUG "instance": "10.142.0.107:9100" 17:38:06 DEBUG }, 17:38:06 DEBUG }, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG "value": [ 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG [...] 17:38:06 DEBUG [...] 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG "pod": "ds-idrepo-2" 17:38:06 DEBUG }, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG 1713544686, 17:38:06 DEBUG "4722.936021611649" 17:38:06 DEBUG 17:38:06 DEBUG ] 17:38:06 DEBUG 17:38:06 DEBUG } 17:38:06 DEBUG "instance": "10.142.0.43:9100" 17:38:06 DEBUG ] 17:38:06 DEBUG }, 17:38:06 DEBUG } 17:38:06 DEBUG "value": [ 17:38:06 DEBUG } 17:38:06 DEBUG 1713544686, 17:38:06 DEBUG "75572014.43333334" 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544686" 17:38:06 INFO [http_cmd]: http status code OK 17:38:06 DEBUG --- status code --- 17:38:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:06 DEBUG --- http response --- 17:38:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:38:06 DEBUG ----- output ----- 17:38:06 DEBUG { 17:38:06 DEBUG "status": "success", 17:38:06 DEBUG "data": { 17:38:06 DEBUG "resultType": "vector", 17:38:06 DEBUG "result": [ 17:38:06 DEBUG { 17:38:06 DEBUG "metric": { 17:38:06 DEBUG "instance": "10.142.0.107:9100" 17:38:06 DEBUG }, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG [...] 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG "instance": "10.142.0.43:9100" 17:38:06 DEBUG }, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG 1713544686, 17:38:06 DEBUG "75566162.93333334" 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544686" 17:38:06 INFO [http_cmd]: http status code OK 17:38:06 DEBUG --- status code --- 17:38:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:06 DEBUG --- http response --- 17:38:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:38:06 DEBUG ----- output ----- 17:38:06 DEBUG { 17:38:06 DEBUG "status": "success", 17:38:06 DEBUG "data": { 17:38:06 DEBUG "resultType": "vector", 17:38:06 DEBUG "result": [ 17:38:06 DEBUG { 17:38:06 DEBUG "metric": { 17:38:06 DEBUG "pod": "ds-idrepo-0" 17:38:06 DEBUG }, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG [...] 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG "pod": "ds-idrepo-2" 17:38:06 DEBUG }, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG 1713544686, 17:38:06 DEBUG "0" 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544686" 17:38:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544686" 17:38:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544686" 17:38:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544686" 17:38:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544686" 17:38:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544686" 17:38:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544686" 17:38:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544686" 17:38:06 INFO [http_cmd]: http status code OK 17:38:06 DEBUG --- status code --- 17:38:06 INFO [http_cmd]: http status code OK 17:38:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:06 INFO [http_cmd]: http status code OK 17:38:06 INFO [http_cmd]: http status code OK 17:38:06 DEBUG --- status code --- 17:38:06 INFO [http_cmd]: http status code OK 17:38:06 DEBUG --- http response --- 17:38:06 DEBUG --- status code --- 17:38:06 DEBUG --- status code --- 17:38:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:06 INFO [http_cmd]: http status code OK 17:38:06 INFO [http_cmd]: http status code OK 17:38:06 DEBUG --- status code --- 17:38:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:38:06 INFO [http_cmd]: http status code OK 17:38:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:06 DEBUG --- http response --- 17:38:06 DEBUG --- status code --- 17:38:06 DEBUG --- status code --- 17:38:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:06 DEBUG --- http response --- 17:38:06 DEBUG ----- output ----- 17:38:06 DEBUG --- status code --- 17:38:06 DEBUG --- http response --- 17:38:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:38:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:06 DEBUG --- http response --- 17:38:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:38:06 DEBUG { 17:38:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:38:06 DEBUG ----- output ----- 17:38:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:38:06 DEBUG --- http response --- 17:38:06 DEBUG --- http response --- 17:38:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:38:06 DEBUG ----- output ----- 17:38:06 DEBUG "status": "success", 17:38:06 DEBUG --- http response --- 17:38:06 DEBUG { 17:38:06 DEBUG ----- output ----- 17:38:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:38:06 DEBUG ----- output ----- 17:38:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:38:06 DEBUG { 17:38:06 DEBUG "data": { 17:38:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:38:06 DEBUG ----- output ----- 17:38:06 DEBUG { 17:38:06 DEBUG ----- output ----- 17:38:06 DEBUG { 17:38:06 DEBUG ----- output ----- 17:38:06 DEBUG "status": "success", 17:38:06 DEBUG "resultType": "vector", 17:38:06 DEBUG "status": "success", 17:38:06 DEBUG { 17:38:06 DEBUG "status": "success", 17:38:06 DEBUG { 17:38:06 DEBUG "status": "success", 17:38:06 DEBUG { 17:38:06 DEBUG "data": { 17:38:06 DEBUG "resultType": "vector", 17:38:06 DEBUG "data": { 17:38:06 DEBUG "status": "success", 17:38:06 DEBUG "data": { 17:38:06 DEBUG "status": "success", 17:38:06 DEBUG "data": { 17:38:06 DEBUG "status": "success", 17:38:06 DEBUG "result": [ 17:38:06 DEBUG "result": [ 17:38:06 DEBUG "resultType": "vector", 17:38:06 DEBUG "data": { 17:38:06 DEBUG "resultType": "vector", 17:38:06 DEBUG "result": [ 17:38:06 DEBUG "resultType": "vector", 17:38:06 DEBUG "data": { 17:38:06 DEBUG { 17:38:06 DEBUG { 17:38:06 DEBUG "result": [ 17:38:06 DEBUG "resultType": "vector", 17:38:06 DEBUG "data": { 17:38:06 DEBUG { 17:38:06 DEBUG "result": [ 17:38:06 DEBUG "resultType": "vector", 17:38:06 DEBUG "metric": { 17:38:06 DEBUG "metric": { 17:38:06 DEBUG { 17:38:06 DEBUG "result": [ 17:38:06 DEBUG "resultType": "vector", 17:38:06 DEBUG "metric": { 17:38:06 DEBUG { 17:38:06 DEBUG "result": [ 17:38:06 DEBUG "pod": "ds-idrepo-0" 17:38:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:38:06 DEBUG "metric": { 17:38:06 DEBUG { 17:38:06 DEBUG "result": [ 17:38:06 DEBUG "instance": "10.142.0.107:9100" 17:38:06 DEBUG "metric": { 17:38:06 DEBUG { 17:38:06 DEBUG }, 17:38:06 DEBUG }, 17:38:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:38:06 DEBUG "metric": { 17:38:06 DEBUG { 17:38:06 DEBUG }, 17:38:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:38:06 DEBUG "metric": { 17:38:06 DEBUG "value": [ 17:38:06 DEBUG "value": [ 17:38:06 DEBUG }, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG "metric": { 17:38:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:38:06 DEBUG }, 17:38:06 DEBUG "pod": "ds-cts-0" 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG "pod": "ds-idrepo-0" 17:38:06 DEBUG "value": [ 17:38:06 DEBUG }, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG }, 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG }, 17:38:06 DEBUG 17:38:06 DEBUG "value": [ 17:38:06 DEBUG 17:38:06 DEBUG "value": [ 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG "value": [ 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG [...] 17:38:06 DEBUG 17:38:06 DEBUG [...] 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG [...] 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG [...] 17:38:06 DEBUG 17:38:06 DEBUG [...] 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG [...] 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG [...] 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG "pod": "ds-idrepo-2" 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG [...] 17:38:06 DEBUG }, 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:38:06 DEBUG }, 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG 17:38:06 DEBUG "value": [ 17:38:06 DEBUG "instance": "10.142.0.43:9100" 17:38:06 DEBUG 17:38:06 DEBUG }, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:38:06 DEBUG "pod": "ds-cts-2" 17:38:06 DEBUG 17:38:06 DEBUG 1713544686, 17:38:06 DEBUG }, 17:38:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:38:06 DEBUG 1713544686, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG }, 17:38:06 DEBUG }, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG "0.023" 17:38:06 DEBUG "value": [ 17:38:06 DEBUG }, 17:38:06 DEBUG "8.02865195376572" 17:38:06 DEBUG 1713544686, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG 1713544686, 17:38:06 DEBUG 17:38:06 DEBUG ] 17:38:06 DEBUG 1713544686, 17:38:06 DEBUG "value": [ 17:38:06 DEBUG ] 17:38:06 DEBUG "7.084297753179398" 17:38:06 DEBUG 1713544686, 17:38:06 DEBUG "321.56666666666666" 17:38:06 DEBUG "pod": "ds-idrepo-2" 17:38:06 DEBUG } 17:38:06 DEBUG ] 17:38:06 DEBUG 1713544686, 17:38:06 DEBUG } 17:38:06 DEBUG ] 17:38:06 DEBUG "2505.633333333333" 17:38:06 DEBUG ] 17:38:06 DEBUG }, 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:06 DEBUG "2505.9333333333334" 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG ] 17:38:06 DEBUG "0.8713666666667212" 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:06 DEBUG "value": [ 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG 1713544686, 17:38:06 DEBUG } 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:06 DEBUG "0" 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:06 DEBUG ] 17:38:06 DEBUG } 17:38:06 DEBUG } 17:38:07 INFO 17:38:07 INFO [loop_until]: kubectl --namespace=xlou top pods 17:38:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:38:07 INFO [loop_until]: OK (rc = 0) 17:38:07 DEBUG --- stdout --- 17:38:07 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17398m 8976Mi am-6fd554f8f7-btd4s 17963m 9009Mi am-6fd554f8f7-wxcsj 18544m 8998Mi ds-cts-0 21371m 12877Mi ds-cts-1 20988m 12890Mi ds-cts-2 22100m 13012Mi ds-idrepo-0 6957m 3423Mi ds-idrepo-1 7097m 3391Mi ds-idrepo-2 7029m 3461Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7868m 1267Mi 17:38:07 DEBUG --- stderr --- 17:38:07 DEBUG 17:38:07 INFO 17:38:07 INFO [loop_until]: kubectl --namespace=xlou top node 17:38:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:38:07 INFO [loop_until]: OK (rc = 0) 17:38:07 DEBUG --- stdout --- 17:38:07 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6180Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7380m 24% 6999Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 50m 0% 2686Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17266m 57% 12271Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 21252m 71% 15923Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5228Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17323m 58% 11899Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16786m 56% 3018Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19265m 64% 11221Mi 10% gke-xlou-cdm-ds-0f601751-j488 7633m 25% 6697Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21452m 71% 17593Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7428m 24% 7218Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21887m 73% 15475Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8181m 51% 5016Mi 8% 17:38:07 DEBUG --- stderr --- 17:38:07 DEBUG 17:39:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544744" 17:39:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544744" 17:39:04 ERROR [http_cmd]: ERROR 17:39:04 ERROR --- status code --- 17:39:04 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:39:04 ERROR --- http response --- 17:39:04 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:39:04 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:39:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544744" 17:39:04 INFO [http_cmd]: http status code OK 17:39:04 DEBUG --- status code --- 17:39:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:04 DEBUG --- http response --- 17:39:04 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:39:04 INFO [http_cmd]: http status code OK 17:39:04 DEBUG --- status code --- 17:39:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:04 DEBUG --- http response --- 17:39:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:39:04 DEBUG ----- output ----- 17:39:04 DEBUG { 17:39:04 DEBUG "status": "success", 17:39:04 DEBUG "data": { 17:39:04 DEBUG "resultType": "vector", 17:39:04 DEBUG "result": [ 17:39:04 DEBUG { 17:39:04 DEBUG "metric": { 17:39:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:39:04 DEBUG }, 17:39:04 DEBUG "value": [ 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG [...] 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:39:04 DEBUG }, 17:39:04 DEBUG "value": [ 17:39:04 DEBUG 1713544744, 17:39:04 DEBUG "2320.233333333333" 17:39:04 DEBUG ] 17:39:04 DEBUG } 17:39:04 DEBUG ] 17:39:04 DEBUG } 17:39:04 DEBUG } 17:39:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544744" 17:39:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544744" 17:39:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544744" 17:39:04 INFO [http_cmd]: http status code OK 17:39:04 INFO [http_cmd]: http status code OK 17:39:04 DEBUG --- status code --- 17:39:04 DEBUG --- status code --- 17:39:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:04 INFO [http_cmd]: http status code OK 17:39:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:04 DEBUG --- http response --- 17:39:04 DEBUG --- status code --- 17:39:04 DEBUG --- http response --- 17:39:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:39:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:39:04 DEBUG --- http response --- 17:39:04 DEBUG ----- output ----- 17:39:04 DEBUG ----- output ----- 17:39:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:39:04 DEBUG { 17:39:04 DEBUG { 17:39:04 DEBUG ----- output ----- 17:39:04 DEBUG "status": "success", 17:39:04 DEBUG "status": "success", 17:39:04 DEBUG { 17:39:04 DEBUG "data": { 17:39:04 DEBUG "data": { 17:39:04 DEBUG "status": "success", 17:39:04 DEBUG "resultType": "vector", 17:39:04 DEBUG "resultType": "vector", 17:39:04 DEBUG "data": { 17:39:04 DEBUG "result": [ 17:39:04 DEBUG "result": [ 17:39:04 DEBUG "resultType": "vector", 17:39:04 DEBUG { 17:39:04 DEBUG { 17:39:04 DEBUG "result": [ 17:39:04 DEBUG "metric": { 17:39:04 DEBUG "metric": { 17:39:04 DEBUG { 17:39:04 DEBUG "pod": "ds-idrepo-0" 17:39:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:39:04 DEBUG "metric": { 17:39:04 DEBUG }, 17:39:04 DEBUG }, 17:39:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:39:04 DEBUG "value": [ 17:39:04 DEBUG "value": [ 17:39:04 DEBUG }, 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG "value": [ 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG [...] 17:39:04 DEBUG [...] 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG [...] 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG "pod": "ds-cts-2" 17:39:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:39:04 DEBUG 17:39:04 DEBUG }, 17:39:04 DEBUG }, 17:39:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:39:04 DEBUG "value": [ 17:39:04 DEBUG "value": [ 17:39:04 DEBUG }, 17:39:04 DEBUG 1713544744, 17:39:04 DEBUG "0" 17:39:04 DEBUG 1713544744, 17:39:04 DEBUG "value": [ 17:39:04 DEBUG ] 17:39:04 DEBUG "0" 17:39:04 DEBUG 1713544744, 17:39:04 DEBUG } 17:39:04 DEBUG ] 17:39:04 DEBUG "2491.0333333333333" 17:39:04 DEBUG ] 17:39:04 DEBUG } 17:39:04 DEBUG ] 17:39:04 DEBUG } 17:39:04 DEBUG ] 17:39:04 DEBUG } 17:39:04 DEBUG } 17:39:04 DEBUG } 17:39:04 DEBUG ] 17:39:04 DEBUG } 17:39:04 DEBUG } 17:39:04 DEBUG } 17:39:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544744" 17:39:04 INFO [http_cmd]: http status code OK 17:39:04 DEBUG --- status code --- 17:39:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:04 DEBUG --- http response --- 17:39:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:39:04 DEBUG ----- output ----- 17:39:04 DEBUG { 17:39:04 DEBUG "status": "success", 17:39:04 DEBUG "data": { 17:39:04 DEBUG "resultType": "vector", 17:39:04 DEBUG "result": [ 17:39:04 DEBUG { 17:39:04 DEBUG "metric": { 17:39:04 DEBUG "pod": "ds-idrepo-1" 17:39:04 DEBUG }, 17:39:04 DEBUG "value": [ 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG [...] 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG "pod": "idm-b85f9d978-txpwv" 17:39:04 DEBUG }, 17:39:04 DEBUG "value": [ 17:39:04 DEBUG 1713544744, 17:39:04 DEBUG "3440.686558681444" 17:39:04 DEBUG ] 17:39:04 DEBUG } 17:39:04 DEBUG ] 17:39:04 DEBUG } 17:39:04 DEBUG } 17:39:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544744" 17:39:04 INFO [http_cmd]: http status code OK 17:39:04 DEBUG --- status code --- 17:39:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:04 DEBUG --- http response --- 17:39:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:39:04 DEBUG ----- output ----- 17:39:04 DEBUG { 17:39:04 DEBUG "status": "success", 17:39:04 DEBUG "data": { 17:39:04 DEBUG "resultType": "vector", 17:39:04 DEBUG "result": [ 17:39:04 DEBUG { 17:39:04 DEBUG "metric": { 17:39:04 DEBUG "instance": "10.142.0.107:9100" 17:39:04 DEBUG }, 17:39:04 DEBUG "value": [ 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG [...] 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG 17:39:04 DEBUG "instance": "10.142.0.43:9100" 17:39:04 DEBUG }, 17:39:04 DEBUG "value": [ 17:39:04 DEBUG 1713544744, 17:39:04 DEBUG "0.4853333333333371" 17:39:04 DEBUG ] 17:39:04 DEBUG } 17:39:04 DEBUG ] 17:39:04 DEBUG } 17:39:04 DEBUG } 17:39:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544745" 17:39:05 INFO [http_cmd]: http status code OK 17:39:05 DEBUG --- status code --- 17:39:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:05 DEBUG --- http response --- 17:39:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:39:05 DEBUG ----- output ----- 17:39:05 DEBUG { 17:39:05 DEBUG "status": "success", 17:39:05 DEBUG "data": { 17:39:05 DEBUG "resultType": "vector", 17:39:05 DEBUG "result": [ 17:39:05 DEBUG { 17:39:05 DEBUG "metric": { 17:39:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG [...] 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 1713544745, 17:39:05 DEBUG "4136988" 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG } 17:39:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544745" 17:39:05 INFO [http_cmd]: http status code OK 17:39:05 DEBUG --- status code --- 17:39:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:05 DEBUG --- http response --- 17:39:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:39:05 DEBUG ----- output ----- 17:39:05 DEBUG { 17:39:05 DEBUG "status": "success", 17:39:05 DEBUG "data": { 17:39:05 DEBUG "resultType": "vector", 17:39:05 DEBUG "result": [ 17:39:05 DEBUG { 17:39:05 DEBUG "metric": { 17:39:05 DEBUG "instance": "10.142.0.107:9100" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG [...] 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG "instance": "10.142.0.43:9100" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 1713544745, 17:39:05 DEBUG "1737330486272" 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG } 17:39:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544745" 17:39:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544745" 17:39:05 INFO [http_cmd]: http status code OK 17:39:05 DEBUG --- status code --- 17:39:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:05 DEBUG --- http response --- 17:39:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:39:05 DEBUG ----- output ----- 17:39:05 DEBUG { 17:39:05 DEBUG "status": "success", 17:39:05 DEBUG "data": { 17:39:05 DEBUG "resultType": "vector", 17:39:05 DEBUG "result": [ 17:39:05 DEBUG { 17:39:05 DEBUG "metric": { 17:39:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG [...] 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 1713544745, 17:39:05 DEBUG "1423396864" 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG } 17:39:05 INFO [http_cmd]: http status code OK 17:39:05 DEBUG --- status code --- 17:39:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:05 DEBUG --- http response --- 17:39:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:39:05 DEBUG ----- output ----- 17:39:05 DEBUG { 17:39:05 DEBUG "status": "success", 17:39:05 DEBUG "data": { 17:39:05 DEBUG "resultType": "vector", 17:39:05 DEBUG "result": [ 17:39:05 DEBUG { 17:39:05 DEBUG "metric": { 17:39:05 DEBUG "pod": "ds-idrepo-0" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG [...] 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG "pod": "ds-idrepo-2" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 1713544745, 17:39:05 DEBUG "0" 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG } 17:39:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544745" 17:39:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544745" 17:39:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544745" 17:39:05 INFO [http_cmd]: http status code OK 17:39:05 DEBUG --- status code --- 17:39:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:05 DEBUG --- http response --- 17:39:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:39:05 DEBUG ----- output ----- 17:39:05 DEBUG { 17:39:05 DEBUG "status": "success", 17:39:05 DEBUG "data": { 17:39:05 DEBUG "resultType": "vector", 17:39:05 DEBUG "result": [ 17:39:05 DEBUG { 17:39:05 DEBUG "metric": { 17:39:05 DEBUG "pod": "ds-cts-0" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG [...] 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG "pod": "ds-cts-2" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 1713544745, 17:39:05 DEBUG "0" 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG } 17:39:05 INFO [http_cmd]: http status code OK 17:39:05 DEBUG --- status code --- 17:39:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:05 DEBUG --- http response --- 17:39:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:39:05 DEBUG ----- output ----- 17:39:05 DEBUG { 17:39:05 DEBUG "status": "success", 17:39:05 DEBUG "data": { 17:39:05 DEBUG "resultType": "vector", 17:39:05 DEBUG "result": [ 17:39:05 DEBUG { 17:39:05 DEBUG "metric": { 17:39:05 DEBUG "pod": "ds-idrepo-0" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG [...] 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG "pod": "ds-idrepo-2" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 1713544745, 17:39:05 DEBUG "0" 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG } 17:39:05 INFO [http_cmd]: http status code OK 17:39:05 DEBUG --- status code --- 17:39:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:05 DEBUG --- http response --- 17:39:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:39:05 DEBUG ----- output ----- 17:39:05 DEBUG { 17:39:05 DEBUG "status": "success", 17:39:05 DEBUG "data": { 17:39:05 DEBUG "resultType": "vector", 17:39:05 DEBUG "result": [ 17:39:05 DEBUG { 17:39:05 DEBUG "metric": { 17:39:05 DEBUG "pod": "ds-cts-0" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG [...] 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG 17:39:05 DEBUG "pod": "ds-cts-2" 17:39:05 DEBUG }, 17:39:05 DEBUG "value": [ 17:39:05 DEBUG 1713544745, 17:39:05 DEBUG "0.1673666666666667" 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG ] 17:39:05 DEBUG } 17:39:05 DEBUG } 17:39:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544746" 17:39:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544746" 17:39:06 INFO [http_cmd]: http status code OK 17:39:06 DEBUG --- status code --- 17:39:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:06 DEBUG --- http response --- 17:39:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:39:06 DEBUG ----- output ----- 17:39:06 DEBUG { 17:39:06 DEBUG "status": "success", 17:39:06 DEBUG "data": { 17:39:06 DEBUG "resultType": "vector", 17:39:06 DEBUG "result": [ 17:39:06 DEBUG { 17:39:06 DEBUG "metric": { 17:39:06 DEBUG "instance": "10.142.0.107:9100" 17:39:06 DEBUG }, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 17:39:06 INFO [http_cmd]: http status code OK 17:39:06 DEBUG 17:39:06 DEBUG --- status code --- 17:39:06 DEBUG 17:39:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:06 DEBUG [...] 17:39:06 DEBUG --- http response --- 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:39:06 DEBUG 17:39:06 DEBUG ----- output ----- 17:39:06 DEBUG "instance": "10.142.0.43:9100" 17:39:06 DEBUG { 17:39:06 DEBUG "status": "success", 17:39:06 DEBUG }, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 1713544746, 17:39:06 DEBUG "75050021.30000001" 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 DEBUG "data": { 17:39:06 DEBUG "resultType": "vector", 17:39:06 DEBUG "result": [ 17:39:06 DEBUG { 17:39:06 DEBUG "metric": { 17:39:06 DEBUG "pod": "ds-idrepo-1" 17:39:06 DEBUG }, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG [...] 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG "pod": "ds-cts-2" 17:39:06 DEBUG }, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 1713544746, 17:39:06 DEBUG "56108504.41100156" 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544746" 17:39:06 INFO [http_cmd]: http status code OK 17:39:06 DEBUG --- status code --- 17:39:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:06 DEBUG --- http response --- 17:39:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:39:06 DEBUG ----- output ----- 17:39:06 DEBUG { 17:39:06 DEBUG "status": "success", 17:39:06 DEBUG "data": { 17:39:06 DEBUG "resultType": "vector", 17:39:06 DEBUG "result": [ 17:39:06 DEBUG { 17:39:06 DEBUG "metric": { 17:39:06 DEBUG "instance": "10.142.0.107:9100" 17:39:06 DEBUG }, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG [...] 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG "instance": "10.142.0.43:9100" 17:39:06 DEBUG }, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 1713544746, 17:39:06 DEBUG "75044936.16666667" 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544746" 17:39:06 INFO [http_cmd]: http status code OK 17:39:06 DEBUG --- status code --- 17:39:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:06 DEBUG --- http response --- 17:39:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:39:06 DEBUG ----- output ----- 17:39:06 DEBUG { 17:39:06 DEBUG "status": "success", 17:39:06 DEBUG "data": { 17:39:06 DEBUG "resultType": "vector", 17:39:06 DEBUG "result": [ 17:39:06 DEBUG { 17:39:06 DEBUG "metric": { 17:39:06 DEBUG "pod": "ds-idrepo-1" 17:39:06 DEBUG }, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG [...] 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG "pod": "ds-cts-2" 17:39:06 DEBUG }, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 1713544746, 17:39:06 DEBUG "61043047.22366373" 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544746" 17:39:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544746" 17:39:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544746" 17:39:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544746" 17:39:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544746" 17:39:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544746" 17:39:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544746" 17:39:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544746" 17:39:06 INFO [http_cmd]: http status code OK 17:39:06 DEBUG --- status code --- 17:39:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:06 DEBUG --- http response --- 17:39:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:39:06 DEBUG ----- output ----- 17:39:06 INFO [http_cmd]: http status code OK 17:39:06 INFO [http_cmd]: http status code OK 17:39:06 INFO [http_cmd]: http status code OK 17:39:06 DEBUG { 17:39:06 INFO [http_cmd]: http status code OK 17:39:06 DEBUG --- status code --- 17:39:06 DEBUG --- status code --- 17:39:06 DEBUG --- status code --- 17:39:06 DEBUG "status": "success", 17:39:06 DEBUG --- status code --- 17:39:06 INFO [http_cmd]: http status code OK 17:39:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:06 DEBUG --- http response --- 17:39:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:06 DEBUG "data": { 17:39:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:06 DEBUG --- status code --- 17:39:06 INFO [http_cmd]: http status code OK 17:39:06 INFO [http_cmd]: http status code OK 17:39:06 DEBUG --- http response --- 17:39:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:39:06 DEBUG --- http response --- 17:39:06 DEBUG "resultType": "vector", 17:39:06 DEBUG --- http response --- 17:39:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:06 DEBUG --- status code --- 17:39:06 DEBUG --- status code --- 17:39:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:39:06 DEBUG ----- output ----- 17:39:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:39:06 DEBUG "result": [ 17:39:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:39:06 DEBUG --- http response --- 17:39:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:39:06 DEBUG ----- output ----- 17:39:06 DEBUG { 17:39:06 DEBUG ----- output ----- 17:39:06 DEBUG { 17:39:06 DEBUG ----- output ----- 17:39:06 DEBUG --- http response --- 17:39:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:39:06 DEBUG --- http response --- 17:39:06 DEBUG { 17:39:06 DEBUG "status": "success", 17:39:06 DEBUG "data": { 17:39:06 DEBUG "metric": { 17:39:06 DEBUG { 17:39:06 DEBUG ----- output ----- 17:39:06 DEBUG { 17:39:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:39:06 DEBUG "status": "success", 17:39:06 DEBUG "resultType": "vector", 17:39:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:39:06 DEBUG "status": "success", 17:39:06 DEBUG "status": "success", 17:39:06 DEBUG { 17:39:06 DEBUG ----- output ----- 17:39:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:39:06 DEBUG "data": { 17:39:06 DEBUG "result": [ 17:39:06 DEBUG }, 17:39:06 DEBUG "data": { 17:39:06 DEBUG "data": { 17:39:06 DEBUG "status": "success", 17:39:06 DEBUG { 17:39:06 DEBUG ----- output ----- 17:39:06 DEBUG "resultType": "vector", 17:39:06 DEBUG { 17:39:06 DEBUG "value": [ 17:39:06 DEBUG "resultType": "vector", 17:39:06 DEBUG "resultType": "vector", 17:39:06 DEBUG "result": [ 17:39:06 DEBUG "status": "success", 17:39:06 DEBUG { 17:39:06 DEBUG "result": [ 17:39:06 DEBUG "metric": { 17:39:06 DEBUG 17:39:06 DEBUG "result": [ 17:39:06 DEBUG "data": { 17:39:06 DEBUG { 17:39:06 DEBUG "metric": { 17:39:06 DEBUG "status": "success", 17:39:06 DEBUG { 17:39:06 DEBUG "pod": "ds-cts-0" 17:39:06 DEBUG 17:39:06 DEBUG { 17:39:06 DEBUG "resultType": "vector", 17:39:06 DEBUG "pod": "ds-idrepo-0" 17:39:06 DEBUG "data": { 17:39:06 DEBUG "data": { 17:39:06 DEBUG "metric": { 17:39:06 DEBUG }, 17:39:06 DEBUG 17:39:06 DEBUG "metric": { 17:39:06 DEBUG "result": [ 17:39:06 DEBUG }, 17:39:06 DEBUG "resultType": "vector", 17:39:06 DEBUG "resultType": "vector", 17:39:06 DEBUG "result": [ 17:39:06 DEBUG "value": [ 17:39:06 DEBUG [...] 17:39:06 DEBUG "pod": "ds-idrepo-1" 17:39:06 DEBUG { 17:39:06 DEBUG "value": [ 17:39:06 DEBUG "result": [ 17:39:06 DEBUG { 17:39:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG }, 17:39:06 DEBUG "metric": { 17:39:06 DEBUG 17:39:06 DEBUG { 17:39:06 DEBUG "metric": { 17:39:06 DEBUG }, 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG "value": [ 17:39:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:39:06 DEBUG 17:39:06 DEBUG "metric": { 17:39:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG }, 17:39:06 DEBUG 17:39:06 DEBUG "instance": "10.142.0.107:9100" 17:39:06 DEBUG }, 17:39:06 DEBUG 17:39:06 DEBUG [...] 17:39:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:39:06 DEBUG 17:39:06 DEBUG "value": [ 17:39:06 DEBUG [...] 17:39:06 DEBUG }, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG }, 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG "value": [ 17:39:06 DEBUG [...] 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG [...] 17:39:06 DEBUG 17:39:06 DEBUG [...] 17:39:06 DEBUG 17:39:06 DEBUG 1713544746, 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG "0.023" 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG [...] 17:39:06 DEBUG 17:39:06 DEBUG "pod": "ds-cts-2" 17:39:06 DEBUG ] 17:39:06 DEBUG 17:39:06 DEBUG "pod": "ds-idrepo-2" 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG 17:39:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:39:06 DEBUG }, 17:39:06 DEBUG } 17:39:06 DEBUG "pod": "ds-cts-2" 17:39:06 DEBUG }, 17:39:06 DEBUG [...] 17:39:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:39:06 DEBUG 17:39:06 DEBUG }, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG ] 17:39:06 DEBUG }, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 17:39:06 DEBUG }, 17:39:06 DEBUG 17:39:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:39:06 DEBUG 1713544746, 17:39:06 DEBUG } 17:39:06 DEBUG "value": [ 17:39:06 DEBUG 1713544746, 17:39:06 DEBUG 17:39:06 DEBUG "value": [ 17:39:06 DEBUG "value": [ 17:39:06 DEBUG }, 17:39:06 DEBUG "318.43333333333334" 17:39:06 DEBUG } 17:39:06 DEBUG 1713544746, 17:39:06 DEBUG "0" 17:39:06 DEBUG 17:39:06 DEBUG "instance": "10.142.0.43:9100" 17:39:06 DEBUG 1713544746, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG ] 17:39:06 DEBUG "21.949414296835577" 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG }, 17:39:06 DEBUG "2499.9" 17:39:06 DEBUG 1713544746, 17:39:06 DEBUG } 17:39:06 DEBUG ] 17:39:06 DEBUG ] 17:39:06 DEBUG 1713544746, 17:39:06 DEBUG "value": [ 17:39:06 DEBUG ] 17:39:06 DEBUG "2494.9333333333334" 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 DEBUG "8.041383388910665" 17:39:06 DEBUG 1713544746, 17:39:06 DEBUG } 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG ] 17:39:06 DEBUG "0.8596000000000287" 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 DEBUG ] 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:06 DEBUG } 17:39:07 INFO 17:39:07 INFO [loop_until]: kubectl --namespace=xlou top pods 17:39:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:39:07 INFO [loop_until]: OK (rc = 0) 17:39:07 DEBUG --- stdout --- 17:39:07 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17503m 8975Mi am-6fd554f8f7-btd4s 17506m 9007Mi am-6fd554f8f7-wxcsj 18714m 9002Mi ds-cts-0 21158m 12730Mi ds-cts-1 21687m 12923Mi ds-cts-2 21573m 12751Mi ds-idrepo-0 7082m 3422Mi ds-idrepo-1 6975m 3391Mi ds-idrepo-2 7220m 3461Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7917m 1269Mi 17:39:07 DEBUG --- stderr --- 17:39:07 DEBUG 17:39:07 INFO 17:39:07 INFO [loop_until]: kubectl --namespace=xlou top node 17:39:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:39:07 INFO [loop_until]: OK (rc = 0) 17:39:07 DEBUG --- stdout --- 17:39:07 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6177Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7451m 24% 7002Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 49m 0% 2682Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17465m 58% 12258Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 21359m 71% 15723Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5227Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17560m 58% 11902Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16830m 56% 3019Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19360m 64% 11228Mi 10% gke-xlou-cdm-ds-0f601751-j488 7710m 25% 6690Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21523m 72% 17701Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7460m 24% 7215Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21846m 73% 15146Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8161m 51% 5027Mi 8% 17:39:07 DEBUG --- stderr --- 17:39:07 DEBUG 17:40:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544804" 17:40:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544804" 17:40:04 ERROR [http_cmd]: ERROR 17:40:04 ERROR --- status code --- 17:40:04 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:40:04 ERROR --- http response --- 17:40:04 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:40:04 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:40:04 INFO [http_cmd]: http status code OK 17:40:04 DEBUG --- status code --- 17:40:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:04 DEBUG --- http response --- 17:40:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544804" 17:40:04 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:40:04 INFO [http_cmd]: http status code OK 17:40:04 DEBUG --- status code --- 17:40:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:04 DEBUG --- http response --- 17:40:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:40:04 DEBUG ----- output ----- 17:40:04 DEBUG { 17:40:04 DEBUG "status": "success", 17:40:04 DEBUG "data": { 17:40:04 DEBUG "resultType": "vector", 17:40:04 DEBUG "result": [ 17:40:04 DEBUG { 17:40:04 DEBUG "metric": { 17:40:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:40:04 DEBUG }, 17:40:04 DEBUG "value": [ 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG [...] 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:40:04 DEBUG }, 17:40:04 DEBUG "value": [ 17:40:04 DEBUG 1713544804, 17:40:04 DEBUG "2322.3" 17:40:04 DEBUG ] 17:40:04 DEBUG } 17:40:04 DEBUG ] 17:40:04 DEBUG } 17:40:04 DEBUG } 17:40:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544804" 17:40:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544804" 17:40:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544804" 17:40:04 INFO [http_cmd]: http status code OK 17:40:04 DEBUG --- status code --- 17:40:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:04 DEBUG --- http response --- 17:40:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:40:04 DEBUG ----- output ----- 17:40:04 DEBUG { 17:40:04 DEBUG "status": "success", 17:40:04 DEBUG "data": { 17:40:04 DEBUG "resultType": "vector", 17:40:04 DEBUG "result": [ 17:40:04 DEBUG { 17:40:04 DEBUG "metric": { 17:40:04 DEBUG "pod": "ds-idrepo-0" 17:40:04 DEBUG }, 17:40:04 DEBUG "value": [ 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG [...] 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG "pod": "ds-cts-2" 17:40:04 DEBUG }, 17:40:04 DEBUG "value": [ 17:40:04 DEBUG 1713544804, 17:40:04 DEBUG "0" 17:40:04 DEBUG ] 17:40:04 DEBUG } 17:40:04 DEBUG ] 17:40:04 DEBUG } 17:40:04 DEBUG } 17:40:04 INFO [http_cmd]: http status code OK 17:40:04 DEBUG --- status code --- 17:40:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:04 INFO [http_cmd]: http status code OK 17:40:04 DEBUG --- http response --- 17:40:04 DEBUG --- status code --- 17:40:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:40:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:04 DEBUG ----- output ----- 17:40:04 DEBUG --- http response --- 17:40:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:40:04 DEBUG ----- output ----- 17:40:04 DEBUG { 17:40:04 DEBUG { 17:40:04 DEBUG "status": "success", 17:40:04 DEBUG "status": "success", 17:40:04 DEBUG "data": { 17:40:04 DEBUG "data": { 17:40:04 DEBUG "resultType": "vector", 17:40:04 DEBUG "resultType": "vector", 17:40:04 DEBUG "result": [ 17:40:04 DEBUG "result": [ 17:40:04 DEBUG { 17:40:04 DEBUG { 17:40:04 DEBUG "metric": { 17:40:04 DEBUG "metric": { 17:40:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:40:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:40:04 DEBUG }, 17:40:04 DEBUG "value": [ 17:40:04 DEBUG }, 17:40:04 DEBUG 17:40:04 DEBUG "value": [ 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG [...] 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG [...] 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:40:04 DEBUG 17:40:04 DEBUG }, 17:40:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:40:04 DEBUG "value": [ 17:40:04 DEBUG }, 17:40:04 DEBUG "value": [ 17:40:04 DEBUG 1713544804, 17:40:04 DEBUG 1713544804, 17:40:04 DEBUG "2489" 17:40:04 DEBUG "0" 17:40:04 DEBUG ] 17:40:04 DEBUG ] 17:40:04 DEBUG } 17:40:04 DEBUG ] 17:40:04 DEBUG } 17:40:04 DEBUG } 17:40:04 DEBUG } 17:40:04 DEBUG ] 17:40:04 DEBUG } 17:40:04 DEBUG } 17:40:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544804" 17:40:04 INFO [http_cmd]: http status code OK 17:40:04 DEBUG --- status code --- 17:40:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:04 DEBUG --- http response --- 17:40:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:40:04 DEBUG ----- output ----- 17:40:04 DEBUG { 17:40:04 DEBUG "status": "success", 17:40:04 DEBUG "data": { 17:40:04 DEBUG "resultType": "vector", 17:40:04 DEBUG "result": [ 17:40:04 DEBUG { 17:40:04 DEBUG "metric": { 17:40:04 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:40:04 DEBUG }, 17:40:04 DEBUG "value": [ 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG [...] 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG "pod": "am-6fd554f8f7-btd4s" 17:40:04 DEBUG }, 17:40:04 DEBUG "value": [ 17:40:04 DEBUG 1713544804, 17:40:04 DEBUG "21354.511698545743" 17:40:04 DEBUG ] 17:40:04 DEBUG } 17:40:04 DEBUG ] 17:40:04 DEBUG } 17:40:04 DEBUG } 17:40:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544804" 17:40:04 INFO [http_cmd]: http status code OK 17:40:04 DEBUG --- status code --- 17:40:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:04 DEBUG --- http response --- 17:40:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:40:04 DEBUG ----- output ----- 17:40:04 DEBUG { 17:40:04 DEBUG "status": "success", 17:40:04 DEBUG "data": { 17:40:04 DEBUG "resultType": "vector", 17:40:04 DEBUG "result": [ 17:40:04 DEBUG { 17:40:04 DEBUG "metric": { 17:40:04 DEBUG "instance": "10.142.0.107:9100" 17:40:04 DEBUG }, 17:40:04 DEBUG "value": [ 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG [...] 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG 17:40:04 DEBUG "instance": "10.142.0.43:9100" 17:40:04 DEBUG }, 17:40:04 DEBUG "value": [ 17:40:04 DEBUG 1713544804, 17:40:04 DEBUG "0.6133333333333364" 17:40:04 DEBUG ] 17:40:04 DEBUG } 17:40:04 DEBUG ] 17:40:04 DEBUG } 17:40:04 DEBUG } 17:40:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544805" 17:40:05 INFO [http_cmd]: http status code OK 17:40:05 DEBUG --- status code --- 17:40:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:05 DEBUG --- http response --- 17:40:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:40:05 DEBUG ----- output ----- 17:40:05 DEBUG { 17:40:05 DEBUG "status": "success", 17:40:05 DEBUG "data": { 17:40:05 DEBUG "resultType": "vector", 17:40:05 DEBUG "result": [ 17:40:05 DEBUG { 17:40:05 DEBUG "metric": { 17:40:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:40:05 DEBUG }, 17:40:05 DEBUG "value": [ 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG [...] 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:40:05 DEBUG }, 17:40:05 DEBUG "value": [ 17:40:05 DEBUG 1713544805, 17:40:05 DEBUG "4147109" 17:40:05 DEBUG ] 17:40:05 DEBUG } 17:40:05 DEBUG ] 17:40:05 DEBUG } 17:40:05 DEBUG } 17:40:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544805" 17:40:05 INFO [http_cmd]: http status code OK 17:40:05 DEBUG --- status code --- 17:40:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:05 DEBUG --- http response --- 17:40:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:40:05 DEBUG ----- output ----- 17:40:05 DEBUG { 17:40:05 DEBUG "status": "success", 17:40:05 DEBUG "data": { 17:40:05 DEBUG "resultType": "vector", 17:40:05 DEBUG "result": [ 17:40:05 DEBUG { 17:40:05 DEBUG "metric": { 17:40:05 DEBUG "instance": "10.142.0.107:9100" 17:40:05 DEBUG }, 17:40:05 DEBUG "value": [ 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG [...] 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG "instance": "10.142.0.43:9100" 17:40:05 DEBUG }, 17:40:05 DEBUG "value": [ 17:40:05 DEBUG 1713544805, 17:40:05 DEBUG "1740849417216" 17:40:05 DEBUG ] 17:40:05 DEBUG } 17:40:05 DEBUG ] 17:40:05 DEBUG } 17:40:05 DEBUG } 17:40:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544805" 17:40:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544805" 17:40:05 INFO [http_cmd]: http status code OK 17:40:05 DEBUG --- status code --- 17:40:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:05 DEBUG --- http response --- 17:40:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:40:05 DEBUG ----- output ----- 17:40:05 DEBUG { 17:40:05 DEBUG "status": "success", 17:40:05 DEBUG "data": { 17:40:05 DEBUG "resultType": "vector", 17:40:05 DEBUG "result": [ 17:40:05 DEBUG { 17:40:05 DEBUG "metric": { 17:40:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:40:05 DEBUG }, 17:40:05 DEBUG "value": [ 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG [...] 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:40:05 DEBUG }, 17:40:05 DEBUG "value": [ 17:40:05 DEBUG 1713544805, 17:40:05 DEBUG "1429057536" 17:40:05 DEBUG ] 17:40:05 DEBUG } 17:40:05 DEBUG ] 17:40:05 DEBUG } 17:40:05 DEBUG } 17:40:05 INFO [http_cmd]: http status code OK 17:40:05 DEBUG --- status code --- 17:40:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:05 DEBUG --- http response --- 17:40:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:40:05 DEBUG ----- output ----- 17:40:05 DEBUG { 17:40:05 DEBUG "status": "success", 17:40:05 DEBUG "data": { 17:40:05 DEBUG "resultType": "vector", 17:40:05 DEBUG "result": [ 17:40:05 DEBUG { 17:40:05 DEBUG "metric": { 17:40:05 DEBUG "pod": "ds-idrepo-0" 17:40:05 DEBUG }, 17:40:05 DEBUG "value": [ 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG [...] 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG 17:40:05 DEBUG "pod": "ds-idrepo-2" 17:40:05 DEBUG }, 17:40:05 DEBUG "value": [ 17:40:05 DEBUG 1713544805, 17:40:05 DEBUG "0" 17:40:05 DEBUG ] 17:40:05 DEBUG } 17:40:05 DEBUG ] 17:40:05 DEBUG } 17:40:05 DEBUG } 17:40:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544806" 17:40:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544806" 17:40:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544806" 17:40:06 INFO [http_cmd]: http status code OK 17:40:06 DEBUG --- status code --- 17:40:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:06 DEBUG --- http response --- 17:40:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:40:06 DEBUG ----- output ----- 17:40:06 DEBUG { 17:40:06 DEBUG "status": "success", 17:40:06 DEBUG "data": { 17:40:06 DEBUG "resultType": "vector", 17:40:06 DEBUG "result": [ 17:40:06 DEBUG { 17:40:06 DEBUG "metric": { 17:40:06 DEBUG "pod": "ds-cts-0" 17:40:06 DEBUG }, 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG [...] 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG "pod": "ds-cts-2" 17:40:06 DEBUG }, 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 1713544806, 17:40:06 DEBUG "11.8" 17:40:06 DEBUG ] 17:40:06 DEBUG } 17:40:06 INFO [http_cmd]: http status code OK 17:40:06 DEBUG ] 17:40:06 DEBUG --- status code --- 17:40:06 DEBUG } 17:40:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:06 DEBUG } 17:40:06 DEBUG --- http response --- 17:40:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:40:06 DEBUG ----- output ----- 17:40:06 DEBUG { 17:40:06 DEBUG "status": "success", 17:40:06 DEBUG "data": { 17:40:06 DEBUG "resultType": "vector", 17:40:06 DEBUG "result": [ 17:40:06 DEBUG { 17:40:06 DEBUG "metric": { 17:40:06 DEBUG "pod": "ds-idrepo-0" 17:40:06 DEBUG }, 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG [...] 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG "pod": "ds-idrepo-2" 17:40:06 DEBUG }, 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 1713544806, 17:40:06 DEBUG "0" 17:40:06 DEBUG ] 17:40:06 DEBUG } 17:40:06 DEBUG ] 17:40:06 DEBUG } 17:40:06 DEBUG } 17:40:06 INFO [http_cmd]: http status code OK 17:40:06 DEBUG --- status code --- 17:40:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:06 DEBUG --- http response --- 17:40:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:40:06 DEBUG ----- output ----- 17:40:06 DEBUG { 17:40:06 DEBUG "status": "success", 17:40:06 DEBUG "data": { 17:40:06 DEBUG "resultType": "vector", 17:40:06 DEBUG "result": [ 17:40:06 DEBUG { 17:40:06 DEBUG "metric": { 17:40:06 DEBUG "pod": "ds-cts-0" 17:40:06 DEBUG }, 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG [...] 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG "pod": "ds-cts-2" 17:40:06 DEBUG }, 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 1713544806, 17:40:06 DEBUG "0.2041999999999999" 17:40:06 DEBUG ] 17:40:06 DEBUG } 17:40:06 DEBUG ] 17:40:06 DEBUG } 17:40:06 DEBUG } 17:40:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544806" 17:40:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544806" 17:40:06 INFO [http_cmd]: http status code OK 17:40:06 DEBUG --- status code --- 17:40:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:06 INFO [http_cmd]: http status code OK 17:40:06 DEBUG --- http response --- 17:40:06 DEBUG --- status code --- 17:40:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:40:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:06 DEBUG ----- output ----- 17:40:06 DEBUG --- http response --- 17:40:06 DEBUG { 17:40:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:40:06 DEBUG "status": "success", 17:40:06 DEBUG ----- output ----- 17:40:06 DEBUG "data": { 17:40:06 DEBUG { 17:40:06 DEBUG "resultType": "vector", 17:40:06 DEBUG "status": "success", 17:40:06 DEBUG "result": [ 17:40:06 DEBUG "data": { 17:40:06 DEBUG { 17:40:06 DEBUG "resultType": "vector", 17:40:06 DEBUG "metric": { 17:40:06 DEBUG "result": [ 17:40:06 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:40:06 DEBUG { 17:40:06 DEBUG }, 17:40:06 DEBUG "metric": { 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 17:40:06 DEBUG "instance": "10.142.0.107:9100" 17:40:06 DEBUG 17:40:06 DEBUG }, 17:40:06 DEBUG 17:40:06 DEBUG "value": [ 17:40:06 DEBUG [...] 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:40:06 DEBUG 17:40:06 DEBUG }, 17:40:06 DEBUG [...] 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 17:40:06 DEBUG 1713544806, 17:40:06 DEBUG 17:40:06 DEBUG "417.8454842219804" 17:40:06 DEBUG ] 17:40:06 DEBUG } 17:40:06 DEBUG 17:40:06 DEBUG ] 17:40:06 DEBUG "instance": "10.142.0.43:9100" 17:40:06 DEBUG } 17:40:06 DEBUG }, 17:40:06 DEBUG } 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 1713544806, 17:40:06 DEBUG "75004051.06666666" 17:40:06 DEBUG ] 17:40:06 DEBUG } 17:40:06 DEBUG ] 17:40:06 DEBUG } 17:40:06 DEBUG } 17:40:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544806" 17:40:06 INFO [http_cmd]: http status code OK 17:40:06 DEBUG --- status code --- 17:40:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:06 DEBUG --- http response --- 17:40:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:40:06 DEBUG ----- output ----- 17:40:06 DEBUG { 17:40:06 DEBUG "status": "success", 17:40:06 DEBUG "data": { 17:40:06 DEBUG "resultType": "vector", 17:40:06 DEBUG "result": [ 17:40:06 DEBUG { 17:40:06 DEBUG "metric": { 17:40:06 DEBUG "instance": "10.142.0.107:9100" 17:40:06 DEBUG }, 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG [...] 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG "instance": "10.142.0.43:9100" 17:40:06 DEBUG }, 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 1713544806, 17:40:06 DEBUG "74999302.63333334" 17:40:06 DEBUG ] 17:40:06 DEBUG } 17:40:06 DEBUG ] 17:40:06 DEBUG } 17:40:06 DEBUG } 17:40:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544806" 17:40:06 INFO [http_cmd]: http status code OK 17:40:06 DEBUG --- status code --- 17:40:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:06 DEBUG --- http response --- 17:40:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:40:06 DEBUG ----- output ----- 17:40:06 DEBUG { 17:40:06 DEBUG "status": "success", 17:40:06 DEBUG "data": { 17:40:06 DEBUG "resultType": "vector", 17:40:06 DEBUG "result": [ 17:40:06 DEBUG { 17:40:06 DEBUG "metric": { 17:40:06 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:40:06 DEBUG }, 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG [...] 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG 17:40:06 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:40:06 DEBUG }, 17:40:06 DEBUG "value": [ 17:40:06 DEBUG 1713544806, 17:40:06 DEBUG "0" 17:40:06 DEBUG ] 17:40:06 DEBUG } 17:40:06 DEBUG ] 17:40:06 DEBUG } 17:40:06 DEBUG } 17:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544807" 17:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544807" 17:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544807" 17:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544807" 17:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544807" 17:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544807" 17:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544807" 17:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544807" 17:40:07 INFO [http_cmd]: http status code OK 17:40:07 INFO [http_cmd]: http status code OK 17:40:07 DEBUG --- status code --- 17:40:07 DEBUG --- status code --- 17:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:07 INFO [http_cmd]: http status code OK 17:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:07 INFO [http_cmd]: http status code OK 17:40:07 INFO [http_cmd]: http status code OK 17:40:07 DEBUG --- http response --- 17:40:07 INFO [http_cmd]: http status code OK 17:40:07 DEBUG --- status code --- 17:40:07 DEBUG --- http response --- 17:40:07 DEBUG --- status code --- 17:40:07 DEBUG --- status code --- 17:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:40:07 DEBUG ----- output ----- 17:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:07 INFO [http_cmd]: http status code OK 17:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:07 INFO [http_cmd]: http status code OK 17:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:07 DEBUG { 17:40:07 DEBUG --- status code --- 17:40:07 DEBUG --- http response --- 17:40:07 DEBUG --- status code --- 17:40:07 DEBUG ----- output ----- 17:40:07 DEBUG --- http response --- 17:40:07 DEBUG --- status code --- 17:40:07 DEBUG --- http response --- 17:40:07 DEBUG "status": "success", 17:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:07 DEBUG { 17:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:40:07 DEBUG "data": { 17:40:07 DEBUG --- http response --- 17:40:07 DEBUG ----- output ----- 17:40:07 DEBUG --- http response --- 17:40:07 DEBUG "status": "success", 17:40:07 DEBUG ----- output ----- 17:40:07 DEBUG --- http response --- 17:40:07 DEBUG ----- output ----- 17:40:07 DEBUG "resultType": "vector", 17:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:40:07 DEBUG { 17:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:40:07 DEBUG "data": { 17:40:07 DEBUG { 17:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:40:07 DEBUG { 17:40:07 DEBUG "result": [ 17:40:07 DEBUG ----- output ----- 17:40:07 DEBUG "status": "success", 17:40:07 DEBUG ----- output ----- 17:40:07 DEBUG "resultType": "vector", 17:40:07 DEBUG "status": "success", 17:40:07 DEBUG ----- output ----- 17:40:07 DEBUG "status": "success", 17:40:07 DEBUG "data": { 17:40:07 DEBUG { 17:40:07 DEBUG "data": { 17:40:07 DEBUG "resultType": "vector", 17:40:07 DEBUG "result": [ 17:40:07 DEBUG "data": { 17:40:07 DEBUG { 17:40:07 DEBUG { 17:40:07 DEBUG "resultType": "vector", 17:40:07 DEBUG "metric": { 17:40:07 DEBUG "result": [ 17:40:07 DEBUG { 17:40:07 DEBUG { 17:40:07 DEBUG "resultType": "vector", 17:40:07 DEBUG "status": "success", 17:40:07 DEBUG "status": "success", 17:40:07 DEBUG "result": [ 17:40:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:40:07 DEBUG { 17:40:07 DEBUG "metric": { 17:40:07 DEBUG "metric": { 17:40:07 DEBUG "result": [ 17:40:07 DEBUG "data": { 17:40:07 DEBUG "data": { 17:40:07 DEBUG { 17:40:07 DEBUG }, 17:40:07 DEBUG "status": "success", 17:40:07 DEBUG "pod": "ds-cts-0" 17:40:07 DEBUG "pod": "ds-idrepo-0" 17:40:07 DEBUG { 17:40:07 DEBUG "resultType": "vector", 17:40:07 DEBUG "resultType": "vector", 17:40:07 DEBUG "metric": { 17:40:07 DEBUG "value": [ 17:40:07 DEBUG "data": { 17:40:07 DEBUG }, 17:40:07 DEBUG }, 17:40:07 DEBUG "metric": { 17:40:07 DEBUG "pod": "ds-idrepo-0" 17:40:07 DEBUG "result": [ 17:40:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:40:07 DEBUG }, 17:40:07 DEBUG "value": [ 17:40:07 DEBUG "value": [ 17:40:07 DEBUG "value": [ 17:40:07 DEBUG }, 17:40:07 DEBUG "result": [ 17:40:07 DEBUG { 17:40:07 DEBUG 17:40:07 DEBUG "resultType": "vector", 17:40:07 DEBUG "result": [ 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG "value": [ 17:40:07 DEBUG { 17:40:07 DEBUG "metric": { 17:40:07 DEBUG 17:40:07 DEBUG { 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG "metric": { 17:40:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:40:07 DEBUG 17:40:07 DEBUG [...] 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG [...] 17:40:07 DEBUG 17:40:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:40:07 DEBUG }, 17:40:07 DEBUG }, 17:40:07 DEBUG "metric": { 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG "value": [ 17:40:07 DEBUG "value": [ 17:40:07 DEBUG "instance": "10.142.0.107:9100" 17:40:07 DEBUG [...] 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG [...] 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG }, 17:40:07 DEBUG 17:40:07 DEBUG [...] 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG "value": [ 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG "pod": "ds-cts-2" 17:40:07 DEBUG 17:40:07 DEBUG }, 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG }, 17:40:07 DEBUG "value": [ 17:40:07 DEBUG 1713544807, 17:40:07 DEBUG [...] 17:40:07 DEBUG [...] 17:40:07 DEBUG 17:40:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:40:07 DEBUG 17:40:07 DEBUG "value": [ 17:40:07 DEBUG "312.3" 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG }, 17:40:07 DEBUG "pod": "ds-idrepo-2" 17:40:07 DEBUG 1713544807, 17:40:07 DEBUG ] 17:40:07 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:40:07 DEBUG 17:40:07 DEBUG 17:40:07 DEBUG [...] 17:40:07 DEBUG "value": [ 17:40:07 DEBUG }, 17:40:07 DEBUG "2488.0333333333333" 17:40:07 DEBUG } 17:40:07 DEBUG }, 17:40:07 DEBUG 17:40:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:40:07 DEBUG 17:40:07 DEBUG 1713544807, 17:40:07 DEBUG "value": [ 17:40:07 DEBUG ] 17:40:07 DEBUG ] 17:40:07 DEBUG "value": [ 17:40:07 DEBUG 17:40:07 DEBUG }, 17:40:07 DEBUG 17:40:07 DEBUG "0.023" 17:40:07 DEBUG 1713544807, 17:40:07 DEBUG } 17:40:07 DEBUG } 17:40:07 DEBUG } 17:40:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:40:07 DEBUG "value": [ 17:40:07 DEBUG 17:40:07 DEBUG ] 17:40:07 DEBUG "0" 17:40:07 DEBUG ] 17:40:07 DEBUG 1713544807, 17:40:07 DEBUG }, 17:40:07 DEBUG 1713544807, 17:40:07 DEBUG "instance": "10.142.0.43:9100" 17:40:07 DEBUG } 17:40:07 DEBUG ] 17:40:07 DEBUG } 17:40:07 DEBUG "0.002937465995649615" 17:40:07 DEBUG "value": [ 17:40:07 DEBUG "8.032134281234818" 17:40:07 DEBUG }, 17:40:07 DEBUG ] 17:40:07 DEBUG } 17:40:07 DEBUG } 17:40:07 DEBUG ] 17:40:07 DEBUG 1713544807, 17:40:07 DEBUG ] 17:40:07 DEBUG "value": [ 17:40:07 DEBUG } 17:40:07 DEBUG ] 17:40:07 DEBUG } 17:40:07 DEBUG "2492.7" 17:40:07 DEBUG } 17:40:07 DEBUG 1713544807, 17:40:07 DEBUG } 17:40:07 DEBUG } 17:40:07 DEBUG ] 17:40:07 DEBUG ] 17:40:07 DEBUG ] 17:40:07 DEBUG "0.9304666666666142" 17:40:07 DEBUG } 17:40:07 DEBUG } 17:40:07 DEBUG } 17:40:07 DEBUG } 17:40:07 DEBUG ] 17:40:07 DEBUG } 17:40:07 DEBUG ] 17:40:07 DEBUG } 17:40:07 DEBUG } 17:40:07 DEBUG } 17:40:07 DEBUG ] 17:40:07 DEBUG } 17:40:07 DEBUG } 17:40:07 DEBUG } 17:40:07 INFO 17:40:07 INFO [loop_until]: kubectl --namespace=xlou top pods 17:40:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:40:07 INFO [loop_until]: OK (rc = 0) 17:40:07 DEBUG --- stdout --- 17:40:07 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17740m 8992Mi am-6fd554f8f7-btd4s 17253m 9032Mi am-6fd554f8f7-wxcsj 19116m 9013Mi ds-cts-0 21399m 12888Mi ds-cts-1 21418m 12675Mi ds-cts-2 22163m 12980Mi ds-idrepo-0 6927m 3423Mi ds-idrepo-1 6769m 3391Mi ds-idrepo-2 6832m 3461Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8192m 1278Mi 17:40:07 DEBUG --- stderr --- 17:40:07 DEBUG 17:40:07 INFO 17:40:07 INFO [loop_until]: kubectl --namespace=xlou top node 17:40:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:40:07 INFO [loop_until]: OK (rc = 0) 17:40:07 DEBUG --- stdout --- 17:40:07 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6182Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7409m 24% 6990Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 51m 0% 2686Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17687m 59% 12290Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 21280m 71% 15870Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 137m 0% 5235Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17677m 59% 11927Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16815m 56% 3034Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19275m 64% 11241Mi 10% gke-xlou-cdm-ds-0f601751-j488 7623m 25% 6697Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21448m 71% 17417Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7425m 24% 7212Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21898m 73% 15415Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7924m 49% 5024Mi 8% 17:40:07 DEBUG --- stderr --- 17:40:07 DEBUG 17:41:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544864" 17:41:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544864" 17:41:04 ERROR [http_cmd]: ERROR 17:41:04 ERROR --- status code --- 17:41:04 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:41:04 ERROR --- http response --- 17:41:04 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:41:04 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:41:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544864" 17:41:04 INFO [http_cmd]: http status code OK 17:41:04 DEBUG --- status code --- 17:41:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:04 DEBUG --- http response --- 17:41:04 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:41:04 INFO [http_cmd]: http status code OK 17:41:04 DEBUG --- status code --- 17:41:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:04 DEBUG --- http response --- 17:41:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:41:04 DEBUG ----- output ----- 17:41:04 DEBUG { 17:41:04 DEBUG "status": "success", 17:41:04 DEBUG "data": { 17:41:04 DEBUG "resultType": "vector", 17:41:04 DEBUG "result": [ 17:41:04 DEBUG { 17:41:04 DEBUG "metric": { 17:41:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:41:04 DEBUG }, 17:41:04 DEBUG "value": [ 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG [...] 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:41:04 DEBUG }, 17:41:04 DEBUG "value": [ 17:41:04 DEBUG 1713544864, 17:41:04 DEBUG "2308.633333333333" 17:41:04 DEBUG ] 17:41:04 DEBUG } 17:41:04 DEBUG ] 17:41:04 DEBUG } 17:41:04 DEBUG } 17:41:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544864" 17:41:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544864" 17:41:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544864" 17:41:04 INFO [http_cmd]: http status code OK 17:41:04 DEBUG --- status code --- 17:41:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:04 DEBUG --- http response --- 17:41:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:41:04 DEBUG ----- output ----- 17:41:04 DEBUG { 17:41:04 DEBUG "status": "success", 17:41:04 DEBUG "data": { 17:41:04 DEBUG "resultType": "vector", 17:41:04 DEBUG "result": [ 17:41:04 DEBUG { 17:41:04 DEBUG "metric": { 17:41:04 DEBUG "pod": "ds-idrepo-0" 17:41:04 DEBUG }, 17:41:04 DEBUG "value": [ 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG [...] 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG "pod": "ds-cts-2" 17:41:04 DEBUG }, 17:41:04 DEBUG "value": [ 17:41:04 DEBUG 1713544864, 17:41:04 DEBUG "0" 17:41:04 DEBUG ] 17:41:04 DEBUG } 17:41:04 DEBUG ] 17:41:04 DEBUG } 17:41:04 DEBUG } 17:41:04 INFO [http_cmd]: http status code OK 17:41:04 DEBUG --- status code --- 17:41:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:04 DEBUG --- http response --- 17:41:04 INFO [http_cmd]: http status code OK 17:41:04 DEBUG --- status code --- 17:41:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:41:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:04 DEBUG ----- output ----- 17:41:04 DEBUG --- http response --- 17:41:04 DEBUG { 17:41:04 DEBUG "status": "success", 17:41:04 DEBUG "data": { 17:41:04 DEBUG "resultType": "vector", 17:41:04 DEBUG "result": [ 17:41:04 DEBUG { 17:41:04 DEBUG "metric": { 17:41:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:41:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:41:04 DEBUG }, 17:41:04 DEBUG ----- output ----- 17:41:04 DEBUG "value": [ 17:41:04 DEBUG { 17:41:04 DEBUG 17:41:04 DEBUG "status": "success", 17:41:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544864" 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG [...] 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:41:04 DEBUG }, 17:41:04 DEBUG "value": [ 17:41:04 DEBUG 1713544864, 17:41:04 DEBUG "data": { 17:41:04 DEBUG "0" 17:41:04 DEBUG "resultType": "vector", 17:41:04 DEBUG ] 17:41:04 DEBUG } 17:41:04 DEBUG ] 17:41:04 DEBUG } 17:41:04 DEBUG "result": [ 17:41:04 DEBUG } 17:41:04 DEBUG { 17:41:04 DEBUG "metric": { 17:41:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:41:04 DEBUG }, 17:41:04 DEBUG "value": [ 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG [...] 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:41:04 DEBUG }, 17:41:04 DEBUG "value": [ 17:41:04 DEBUG 1713544864, 17:41:04 DEBUG "2479.866666666667" 17:41:04 DEBUG ] 17:41:04 DEBUG } 17:41:04 DEBUG ] 17:41:04 DEBUG } 17:41:04 DEBUG } 17:41:04 INFO [http_cmd]: http status code OK 17:41:04 DEBUG --- status code --- 17:41:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:04 DEBUG --- http response --- 17:41:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:41:04 DEBUG ----- output ----- 17:41:04 DEBUG { 17:41:04 DEBUG "status": "success", 17:41:04 DEBUG "data": { 17:41:04 DEBUG "resultType": "vector", 17:41:04 DEBUG "result": [ 17:41:04 DEBUG { 17:41:04 DEBUG "metric": { 17:41:04 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:41:04 DEBUG }, 17:41:04 DEBUG "value": [ 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG [...] 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG "pod": "idm-b85f9d978-txpwv" 17:41:04 DEBUG }, 17:41:04 DEBUG "value": [ 17:41:04 DEBUG 1713544864, 17:41:04 DEBUG "0" 17:41:04 DEBUG ] 17:41:04 DEBUG } 17:41:04 DEBUG ] 17:41:04 DEBUG } 17:41:04 DEBUG } 17:41:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544864" 17:41:04 INFO [http_cmd]: http status code OK 17:41:04 DEBUG --- status code --- 17:41:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:04 DEBUG --- http response --- 17:41:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:41:04 DEBUG ----- output ----- 17:41:04 DEBUG { 17:41:04 DEBUG "status": "success", 17:41:04 DEBUG "data": { 17:41:04 DEBUG "resultType": "vector", 17:41:04 DEBUG "result": [ 17:41:04 DEBUG { 17:41:04 DEBUG "metric": { 17:41:04 DEBUG "instance": "10.142.0.107:9100" 17:41:04 DEBUG }, 17:41:04 DEBUG "value": [ 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG [...] 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG 17:41:04 DEBUG "instance": "10.142.0.43:9100" 17:41:04 DEBUG }, 17:41:04 DEBUG "value": [ 17:41:04 DEBUG 1713544864, 17:41:04 DEBUG "0.637333333333339" 17:41:04 DEBUG ] 17:41:04 DEBUG } 17:41:04 DEBUG ] 17:41:04 DEBUG } 17:41:04 DEBUG } 17:41:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544865" 17:41:05 INFO [http_cmd]: http status code OK 17:41:05 DEBUG --- status code --- 17:41:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:05 DEBUG --- http response --- 17:41:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:41:05 DEBUG ----- output ----- 17:41:05 DEBUG { 17:41:05 DEBUG "status": "success", 17:41:05 DEBUG "data": { 17:41:05 DEBUG "resultType": "vector", 17:41:05 DEBUG "result": [ 17:41:05 DEBUG { 17:41:05 DEBUG "metric": { 17:41:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:41:05 DEBUG }, 17:41:05 DEBUG "value": [ 17:41:05 DEBUG 17:41:05 DEBUG 17:41:05 DEBUG 17:41:05 DEBUG [...] 17:41:05 DEBUG 17:41:05 DEBUG 17:41:05 DEBUG 17:41:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:41:05 DEBUG }, 17:41:05 DEBUG "value": [ 17:41:05 DEBUG 1713544865, 17:41:05 DEBUG "4157142.5" 17:41:05 DEBUG ] 17:41:05 DEBUG } 17:41:05 DEBUG ] 17:41:05 DEBUG } 17:41:05 DEBUG } 17:41:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544865" 17:41:05 INFO [http_cmd]: http status code OK 17:41:05 DEBUG --- status code --- 17:41:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:05 DEBUG --- http response --- 17:41:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:41:05 DEBUG ----- output ----- 17:41:05 DEBUG { 17:41:05 DEBUG "status": "success", 17:41:05 DEBUG "data": { 17:41:05 DEBUG "resultType": "vector", 17:41:05 DEBUG "result": [ 17:41:05 DEBUG { 17:41:05 DEBUG "metric": { 17:41:05 DEBUG "instance": "10.142.0.107:9100" 17:41:05 DEBUG }, 17:41:05 DEBUG "value": [ 17:41:05 DEBUG 17:41:05 DEBUG 17:41:05 DEBUG 17:41:05 DEBUG [...] 17:41:05 DEBUG 17:41:05 DEBUG 17:41:05 DEBUG 17:41:05 DEBUG "instance": "10.142.0.43:9100" 17:41:05 DEBUG }, 17:41:05 DEBUG "value": [ 17:41:05 DEBUG 1713544865, 17:41:05 DEBUG "1744366590976" 17:41:05 DEBUG ] 17:41:05 DEBUG } 17:41:05 DEBUG ] 17:41:05 DEBUG } 17:41:05 DEBUG } 17:41:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544866" 17:41:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544866" 17:41:06 INFO [http_cmd]: http status code OK 17:41:06 DEBUG --- status code --- 17:41:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:06 DEBUG --- http response --- 17:41:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:41:06 DEBUG ----- output ----- 17:41:06 DEBUG { 17:41:06 DEBUG "status": "success", 17:41:06 DEBUG "data": { 17:41:06 DEBUG "resultType": "vector", 17:41:06 DEBUG "result": [ 17:41:06 DEBUG { 17:41:06 DEBUG "metric": { 17:41:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG [...] 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 1713544866, 17:41:06 DEBUG "1432768512" 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG } 17:41:06 INFO [http_cmd]: http status code OK 17:41:06 DEBUG --- status code --- 17:41:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:06 DEBUG --- http response --- 17:41:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:41:06 DEBUG ----- output ----- 17:41:06 DEBUG { 17:41:06 DEBUG "status": "success", 17:41:06 DEBUG "data": { 17:41:06 DEBUG "resultType": "vector", 17:41:06 DEBUG "result": [ 17:41:06 DEBUG { 17:41:06 DEBUG "metric": { 17:41:06 DEBUG "pod": "ds-idrepo-0" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG [...] 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG "pod": "ds-idrepo-2" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 1713544866, 17:41:06 DEBUG "0" 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG } 17:41:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544866" 17:41:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544866" 17:41:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544866" 17:41:06 INFO [http_cmd]: http status code OK 17:41:06 DEBUG --- status code --- 17:41:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:06 DEBUG --- http response --- 17:41:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:41:06 DEBUG ----- output ----- 17:41:06 DEBUG { 17:41:06 DEBUG "status": "success", 17:41:06 DEBUG "data": { 17:41:06 DEBUG "resultType": "vector", 17:41:06 DEBUG "result": [ 17:41:06 DEBUG { 17:41:06 DEBUG "metric": { 17:41:06 DEBUG "pod": "ds-cts-0" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG [...] 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG "pod": "ds-cts-2" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 1713544866, 17:41:06 DEBUG "30.166666666666668" 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG } 17:41:06 INFO [http_cmd]: http status code OK 17:41:06 INFO [http_cmd]: http status code OK 17:41:06 DEBUG --- status code --- 17:41:06 DEBUG --- status code --- 17:41:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:06 DEBUG --- http response --- 17:41:06 DEBUG --- http response --- 17:41:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:41:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:41:06 DEBUG ----- output ----- 17:41:06 DEBUG ----- output ----- 17:41:06 DEBUG { 17:41:06 DEBUG { 17:41:06 DEBUG "status": "success", 17:41:06 DEBUG "status": "success", 17:41:06 DEBUG "data": { 17:41:06 DEBUG "data": { 17:41:06 DEBUG "resultType": "vector", 17:41:06 DEBUG "resultType": "vector", 17:41:06 DEBUG "result": [ 17:41:06 DEBUG "result": [ 17:41:06 DEBUG { 17:41:06 DEBUG "metric": { 17:41:06 DEBUG "pod": "ds-idrepo-0" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG [...] 17:41:06 DEBUG 17:41:06 DEBUG { 17:41:06 DEBUG 17:41:06 DEBUG "metric": { 17:41:06 DEBUG 17:41:06 DEBUG "pod": "ds-cts-0" 17:41:06 DEBUG "pod": "ds-idrepo-2" 17:41:06 DEBUG }, 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 17:41:06 DEBUG 1713544866, 17:41:06 DEBUG 17:41:06 DEBUG "0" 17:41:06 DEBUG 17:41:06 DEBUG ] 17:41:06 DEBUG [...] 17:41:06 DEBUG } 17:41:06 DEBUG 17:41:06 DEBUG ] 17:41:06 DEBUG 17:41:06 DEBUG } 17:41:06 DEBUG 17:41:06 DEBUG } 17:41:06 DEBUG "pod": "ds-cts-2" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 1713544866, 17:41:06 DEBUG "0.2425333333333332" 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG } 17:41:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544866" 17:41:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544866" 17:41:06 INFO [http_cmd]: http status code OK 17:41:06 DEBUG --- status code --- 17:41:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:06 DEBUG --- http response --- 17:41:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:41:06 DEBUG ----- output ----- 17:41:06 DEBUG { 17:41:06 DEBUG "status": "success", 17:41:06 DEBUG "data": { 17:41:06 DEBUG "resultType": "vector", 17:41:06 DEBUG "result": [ 17:41:06 DEBUG { 17:41:06 DEBUG "metric": { 17:41:06 DEBUG "instance": "10.142.0.107:9100" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG [...] 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG "instance": "10.142.0.43:9100" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 1713544866, 17:41:06 DEBUG "74780136.16666667" 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 INFO [http_cmd]: http status code OK 17:41:06 DEBUG ] 17:41:06 DEBUG --- status code --- 17:41:06 DEBUG } 17:41:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:06 DEBUG } 17:41:06 DEBUG --- http response --- 17:41:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:41:06 DEBUG ----- output ----- 17:41:06 DEBUG { 17:41:06 DEBUG "status": "success", 17:41:06 DEBUG "data": { 17:41:06 DEBUG "resultType": "vector", 17:41:06 DEBUG "result": [ 17:41:06 DEBUG { 17:41:06 DEBUG "metric": { 17:41:06 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG [...] 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG "pod": "idm-b85f9d978-txpwv" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 1713544866, 17:41:06 DEBUG "427.5388416426101" 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG } 17:41:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544866" 17:41:06 INFO [http_cmd]: http status code OK 17:41:06 DEBUG --- status code --- 17:41:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:06 DEBUG --- http response --- 17:41:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:41:06 DEBUG ----- output ----- 17:41:06 DEBUG { 17:41:06 DEBUG "status": "success", 17:41:06 DEBUG "data": { 17:41:06 DEBUG "resultType": "vector", 17:41:06 DEBUG "result": [ 17:41:06 DEBUG { 17:41:06 DEBUG "metric": { 17:41:06 DEBUG "instance": "10.142.0.107:9100" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG [...] 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG "instance": "10.142.0.43:9100" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 1713544866, 17:41:06 DEBUG "74773658.23333333" 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG } 17:41:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544866" 17:41:06 INFO [http_cmd]: http status code OK 17:41:06 DEBUG --- status code --- 17:41:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:06 DEBUG --- http response --- 17:41:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:41:06 DEBUG ----- output ----- 17:41:06 DEBUG { 17:41:06 DEBUG "status": "success", 17:41:06 DEBUG "data": { 17:41:06 DEBUG "resultType": "vector", 17:41:06 DEBUG "result": [ 17:41:06 DEBUG { 17:41:06 DEBUG "metric": { 17:41:06 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG [...] 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG 17:41:06 DEBUG "pod": "idm-b85f9d978-txpwv" 17:41:06 DEBUG }, 17:41:06 DEBUG "value": [ 17:41:06 DEBUG 1713544866, 17:41:06 DEBUG "0" 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG ] 17:41:06 DEBUG } 17:41:06 DEBUG } 17:41:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544867" 17:41:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544867" 17:41:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544867" 17:41:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544867" 17:41:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544867" 17:41:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544867" 17:41:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544867" 17:41:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544867" 17:41:07 INFO [http_cmd]: http status code OK 17:41:07 DEBUG --- status code --- 17:41:07 INFO [http_cmd]: http status code OK 17:41:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:07 DEBUG --- status code --- 17:41:07 DEBUG --- http response --- 17:41:07 INFO [http_cmd]: http status code OK 17:41:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:07 DEBUG --- http response --- 17:41:07 INFO [http_cmd]: http status code OK 17:41:07 INFO [http_cmd]: http status code OK 17:41:07 DEBUG --- status code --- 17:41:07 INFO [http_cmd]: http status code OK 17:41:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:41:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:41:07 DEBUG ----- output ----- 17:41:07 DEBUG --- status code --- 17:41:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:07 DEBUG --- http response --- 17:41:07 INFO [http_cmd]: http status code OK 17:41:07 INFO [http_cmd]: http status code OK 17:41:07 DEBUG ----- output ----- 17:41:07 DEBUG --- status code --- 17:41:07 DEBUG { 17:41:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:07 DEBUG --- status code --- 17:41:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:41:07 DEBUG --- status code --- 17:41:07 DEBUG --- status code --- 17:41:07 DEBUG { 17:41:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:07 DEBUG "status": "success", 17:41:07 DEBUG --- http response --- 17:41:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:07 DEBUG ----- output ----- 17:41:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:07 DEBUG --- http response --- 17:41:07 DEBUG "status": "success", 17:41:07 DEBUG --- http response --- 17:41:07 DEBUG "data": { 17:41:07 DEBUG "resultType": "vector", 17:41:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:41:07 DEBUG { 17:41:07 DEBUG "status": "success", 17:41:07 DEBUG "data": { 17:41:07 DEBUG "resultType": "vector", 17:41:07 DEBUG "result": [ 17:41:07 DEBUG --- http response --- 17:41:07 DEBUG "result": [ 17:41:07 DEBUG ----- output ----- 17:41:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:41:07 DEBUG ----- output ----- 17:41:07 DEBUG { 17:41:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:41:07 DEBUG "data": { 17:41:07 DEBUG { 17:41:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:41:07 DEBUG { 17:41:07 DEBUG "status": "success", 17:41:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:41:07 DEBUG ----- output ----- 17:41:07 DEBUG --- http response --- 17:41:07 DEBUG "resultType": "vector", 17:41:07 DEBUG "metric": { 17:41:07 DEBUG ----- output ----- 17:41:07 DEBUG { 17:41:07 DEBUG { 17:41:07 DEBUG { 17:41:07 DEBUG "metric": { 17:41:07 DEBUG "result": [ 17:41:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:41:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:41:07 DEBUG "status": "success", 17:41:07 DEBUG "status": "success", 17:41:07 DEBUG "data": { 17:41:07 DEBUG "resultType": "vector", 17:41:07 DEBUG "pod": "ds-cts-0" 17:41:07 DEBUG { 17:41:07 DEBUG ----- output ----- 17:41:07 DEBUG }, 17:41:07 DEBUG "data": { 17:41:07 DEBUG "data": { 17:41:07 DEBUG "result": [ 17:41:07 DEBUG "status": "success", 17:41:07 DEBUG }, 17:41:07 DEBUG "metric": { 17:41:07 DEBUG { 17:41:07 DEBUG "value": [ 17:41:07 DEBUG "resultType": "vector", 17:41:07 DEBUG "resultType": "vector", 17:41:07 DEBUG { 17:41:07 DEBUG "data": { 17:41:07 DEBUG "value": [ 17:41:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:41:07 DEBUG "status": "success", 17:41:07 DEBUG 17:41:07 DEBUG "result": [ 17:41:07 DEBUG "result": [ 17:41:07 DEBUG "metric": { 17:41:07 DEBUG "resultType": "vector", 17:41:07 DEBUG 17:41:07 DEBUG }, 17:41:07 DEBUG "data": { 17:41:07 DEBUG 17:41:07 DEBUG { 17:41:07 DEBUG { 17:41:07 DEBUG "pod": "ds-idrepo-0" 17:41:07 DEBUG "result": [ 17:41:07 DEBUG 17:41:07 DEBUG "value": [ 17:41:07 DEBUG "resultType": "vector", 17:41:07 DEBUG 17:41:07 DEBUG "metric": { 17:41:07 DEBUG "metric": { 17:41:07 DEBUG }, 17:41:07 DEBUG "value": [ 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG "result": [ 17:41:07 DEBUG [...] 17:41:07 DEBUG "pod": "ds-idrepo-0" 17:41:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG { 17:41:07 DEBUG 17:41:07 DEBUG { 17:41:07 DEBUG 17:41:07 DEBUG }, 17:41:07 DEBUG }, 17:41:07 DEBUG 17:41:07 DEBUG [...] 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG "metric": { 17:41:07 DEBUG 17:41:07 DEBUG "value": [ 17:41:07 DEBUG "value": [ 17:41:07 DEBUG "metric": { 17:41:07 DEBUG 17:41:07 DEBUG [...] 17:41:07 DEBUG 17:41:07 DEBUG "instance": "10.142.0.107:9100" 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG [...] 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG }, 17:41:07 DEBUG "pod": "idm-b85f9d978-txpwv" 17:41:07 DEBUG 17:41:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:41:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG "value": [ 17:41:07 DEBUG }, 17:41:07 DEBUG }, 17:41:07 DEBUG "pod": "ds-cts-2" 17:41:07 DEBUG }, 17:41:07 DEBUG }, 17:41:07 DEBUG "value": [ 17:41:07 DEBUG [...] 17:41:07 DEBUG 17:41:07 DEBUG "value": [ 17:41:07 DEBUG "value": [ 17:41:07 DEBUG }, 17:41:07 DEBUG "value": [ 17:41:07 DEBUG 1713544867, 17:41:07 DEBUG "2480.7" 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG 1713544867, 17:41:07 DEBUG "value": [ 17:41:07 DEBUG ] 17:41:07 DEBUG 17:41:07 DEBUG 1713544867, 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG "0.002147992529453779" 17:41:07 DEBUG 1713544867, 17:41:07 DEBUG 17:41:07 DEBUG } 17:41:07 DEBUG 17:41:07 DEBUG "0.023" 17:41:07 DEBUG ] 17:41:07 DEBUG } 17:41:07 DEBUG ] 17:41:07 DEBUG "316.6333333333333" 17:41:07 DEBUG [...] 17:41:07 DEBUG ] 17:41:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:41:07 DEBUG [...] 17:41:07 DEBUG 17:41:07 DEBUG } 17:41:07 DEBUG ] 17:41:07 DEBUG ] 17:41:07 DEBUG 17:41:07 DEBUG } 17:41:07 DEBUG }, 17:41:07 DEBUG 17:41:07 DEBUG [...] 17:41:07 DEBUG } 17:41:07 DEBUG } 17:41:07 DEBUG ] 17:41:07 DEBUG 17:41:07 DEBUG 17:41:07 DEBUG "pod": "ds-idrepo-2" 17:41:07 DEBUG }, 17:41:07 DEBUG 17:41:07 DEBUG } 17:41:07 DEBUG } 17:41:07 DEBUG ] 17:41:07 DEBUG 17:41:07 DEBUG "value": [ 17:41:07 DEBUG "value": [ 17:41:07 DEBUG 17:41:07 DEBUG } 17:41:07 DEBUG } 17:41:07 DEBUG } 17:41:07 DEBUG 17:41:07 DEBUG 1713544867, 17:41:07 DEBUG 1713544867, 17:41:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:41:07 DEBUG } 17:41:07 DEBUG 17:41:07 DEBUG "0" 17:41:07 DEBUG "2489.8" 17:41:07 DEBUG }, 17:41:07 DEBUG "value": [ 17:41:07 DEBUG 1713544867, 17:41:07 DEBUG ] 17:41:07 DEBUG } 17:41:07 DEBUG "instance": "10.142.0.43:9100" 17:41:07 DEBUG ] 17:41:07 DEBUG "8.03841024841619" 17:41:07 DEBUG }, 17:41:07 DEBUG } 17:41:07 DEBUG ] 17:41:07 DEBUG "value": [ 17:41:07 DEBUG 1713544867, 17:41:07 DEBUG } 17:41:07 DEBUG } 17:41:07 DEBUG "0.9383666666666338" 17:41:07 DEBUG ] 17:41:07 DEBUG ] 17:41:07 DEBUG ] 17:41:07 DEBUG } 17:41:07 DEBUG } 17:41:07 DEBUG } 17:41:07 DEBUG } 17:41:07 DEBUG ] 17:41:07 DEBUG ] 17:41:07 DEBUG } 17:41:07 DEBUG } 17:41:07 DEBUG } 17:41:07 DEBUG } 17:41:07 INFO 17:41:07 INFO [loop_until]: kubectl --namespace=xlou top pods 17:41:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:41:07 INFO [loop_until]: OK (rc = 0) 17:41:07 DEBUG --- stdout --- 17:41:07 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17356m 8997Mi am-6fd554f8f7-btd4s 17417m 9038Mi am-6fd554f8f7-wxcsj 18494m 9025Mi ds-cts-0 21203m 12955Mi ds-cts-1 21508m 13036Mi ds-cts-2 21906m 13143Mi ds-idrepo-0 6962m 3423Mi ds-idrepo-1 6560m 3392Mi ds-idrepo-2 6979m 3461Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7876m 1277Mi 17:41:07 DEBUG --- stderr --- 17:41:07 DEBUG 17:41:07 INFO 17:41:07 INFO [loop_until]: kubectl --namespace=xlou top node 17:41:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:41:07 INFO [loop_until]: OK (rc = 0) 17:41:07 DEBUG --- stdout --- 17:41:07 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6165Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7436m 24% 6982Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 49m 0% 2692Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17483m 58% 12285Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 21233m 71% 15973Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5256Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17310m 57% 11937Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16640m 55% 3022Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19279m 64% 11253Mi 10% gke-xlou-cdm-ds-0f601751-j488 7610m 25% 6696Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21576m 72% 17812Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7431m 24% 7225Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21931m 73% 15583Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8179m 51% 5024Mi 8% 17:41:07 DEBUG --- stderr --- 17:41:07 DEBUG 17:42:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544924" 17:42:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544924" 17:42:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544924" 17:42:04 ERROR [http_cmd]: ERROR 17:42:04 ERROR --- status code --- 17:42:04 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:42:04 ERROR --- http response --- 17:42:04 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:42:04 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:42:04 INFO [http_cmd]: http status code OK 17:42:04 DEBUG --- status code --- 17:42:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:04 DEBUG --- http response --- 17:42:04 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:42:04 INFO [http_cmd]: http status code OK 17:42:04 DEBUG --- status code --- 17:42:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:04 DEBUG --- http response --- 17:42:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:42:04 DEBUG ----- output ----- 17:42:04 DEBUG { 17:42:04 DEBUG "status": "success", 17:42:04 DEBUG "data": { 17:42:04 DEBUG "resultType": "vector", 17:42:04 DEBUG "result": [ 17:42:04 DEBUG { 17:42:04 DEBUG "metric": { 17:42:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:42:04 DEBUG }, 17:42:04 DEBUG "value": [ 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG [...] 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:42:04 DEBUG }, 17:42:04 DEBUG "value": [ 17:42:04 DEBUG 1713544924, 17:42:04 DEBUG "2328.0333333333333" 17:42:04 DEBUG ] 17:42:04 DEBUG } 17:42:04 DEBUG ] 17:42:04 DEBUG } 17:42:04 DEBUG } 17:42:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544924" 17:42:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544924" 17:42:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544924" 17:42:04 INFO [http_cmd]: http status code OK 17:42:04 DEBUG --- status code --- 17:42:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:04 DEBUG --- http response --- 17:42:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:42:04 DEBUG ----- output ----- 17:42:04 DEBUG { 17:42:04 DEBUG "status": "success", 17:42:04 DEBUG "data": { 17:42:04 DEBUG "resultType": "vector", 17:42:04 DEBUG "result": [ 17:42:04 DEBUG { 17:42:04 DEBUG "metric": { 17:42:04 DEBUG "pod": "ds-idrepo-0" 17:42:04 DEBUG }, 17:42:04 DEBUG "value": [ 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG [...] 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG "pod": "ds-cts-2" 17:42:04 DEBUG }, 17:42:04 DEBUG "value": [ 17:42:04 DEBUG 1713544924, 17:42:04 DEBUG "0" 17:42:04 DEBUG ] 17:42:04 DEBUG } 17:42:04 DEBUG ] 17:42:04 DEBUG } 17:42:04 DEBUG } 17:42:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544924" 17:42:04 INFO [http_cmd]: http status code OK 17:42:04 DEBUG --- status code --- 17:42:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:04 DEBUG --- http response --- 17:42:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:42:04 DEBUG ----- output ----- 17:42:04 DEBUG { 17:42:04 DEBUG "status": "success", 17:42:04 DEBUG "data": { 17:42:04 DEBUG "resultType": "vector", 17:42:04 DEBUG "result": [ 17:42:04 DEBUG { 17:42:04 DEBUG "metric": { 17:42:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:42:04 DEBUG }, 17:42:04 DEBUG "value": [ 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG [...] 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:42:04 DEBUG }, 17:42:04 DEBUG "value": [ 17:42:04 DEBUG 1713544924, 17:42:04 DEBUG "0" 17:42:04 DEBUG ] 17:42:04 DEBUG } 17:42:04 DEBUG ] 17:42:04 DEBUG } 17:42:04 DEBUG } 17:42:04 INFO [http_cmd]: http status code OK 17:42:04 DEBUG --- status code --- 17:42:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:04 DEBUG --- http response --- 17:42:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:42:04 DEBUG ----- output ----- 17:42:04 DEBUG { 17:42:04 DEBUG "status": "success", 17:42:04 DEBUG "data": { 17:42:04 DEBUG "resultType": "vector", 17:42:04 DEBUG "result": [ 17:42:04 DEBUG { 17:42:04 DEBUG "metric": { 17:42:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:42:04 DEBUG }, 17:42:04 DEBUG "value": [ 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG [...] 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:42:04 DEBUG }, 17:42:04 DEBUG "value": [ 17:42:04 DEBUG 1713544924, 17:42:04 DEBUG "2493.5333333333333" 17:42:04 DEBUG ] 17:42:04 DEBUG } 17:42:04 DEBUG ] 17:42:04 DEBUG } 17:42:04 DEBUG } 17:42:04 INFO [http_cmd]: http status code OK 17:42:04 DEBUG --- status code --- 17:42:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:04 DEBUG --- http response --- 17:42:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:42:04 DEBUG ----- output ----- 17:42:04 DEBUG { 17:42:04 DEBUG "status": "success", 17:42:04 DEBUG "data": { 17:42:04 DEBUG "resultType": "vector", 17:42:04 DEBUG "result": [ 17:42:04 DEBUG { 17:42:04 DEBUG "metric": { 17:42:04 DEBUG "pod": "ds-idrepo-0" 17:42:04 DEBUG }, 17:42:04 DEBUG "value": [ 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG [...] 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:42:04 DEBUG }, 17:42:04 DEBUG "value": [ 17:42:04 DEBUG 1713544924, 17:42:04 DEBUG "127.81978734057908" 17:42:04 DEBUG ] 17:42:04 DEBUG } 17:42:04 DEBUG ] 17:42:04 DEBUG } 17:42:04 DEBUG } 17:42:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544924" 17:42:04 INFO [http_cmd]: http status code OK 17:42:04 DEBUG --- status code --- 17:42:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:04 DEBUG --- http response --- 17:42:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:42:04 DEBUG ----- output ----- 17:42:04 DEBUG { 17:42:04 DEBUG "status": "success", 17:42:04 DEBUG "data": { 17:42:04 DEBUG "resultType": "vector", 17:42:04 DEBUG "result": [ 17:42:04 DEBUG { 17:42:04 DEBUG "metric": { 17:42:04 DEBUG "instance": "10.142.0.107:9100" 17:42:04 DEBUG }, 17:42:04 DEBUG "value": [ 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG [...] 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG 17:42:04 DEBUG "instance": "10.142.0.43:9100" 17:42:04 DEBUG }, 17:42:04 DEBUG "value": [ 17:42:04 DEBUG 1713544924, 17:42:04 DEBUG "0.6156666666666749" 17:42:04 DEBUG ] 17:42:04 DEBUG } 17:42:04 DEBUG ] 17:42:04 DEBUG } 17:42:04 DEBUG } 17:42:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544925" 17:42:05 INFO [http_cmd]: http status code OK 17:42:05 DEBUG --- status code --- 17:42:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:05 DEBUG --- http response --- 17:42:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:42:05 DEBUG ----- output ----- 17:42:05 DEBUG { 17:42:05 DEBUG "status": "success", 17:42:05 DEBUG "data": { 17:42:05 DEBUG "resultType": "vector", 17:42:05 DEBUG "result": [ 17:42:05 DEBUG { 17:42:05 DEBUG "metric": { 17:42:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:42:05 DEBUG }, 17:42:05 DEBUG "value": [ 17:42:05 DEBUG 17:42:05 DEBUG 17:42:05 DEBUG 17:42:05 DEBUG [...] 17:42:05 DEBUG 17:42:05 DEBUG 17:42:05 DEBUG 17:42:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:42:05 DEBUG }, 17:42:05 DEBUG "value": [ 17:42:05 DEBUG 1713544925, 17:42:05 DEBUG "4166604.5" 17:42:05 DEBUG ] 17:42:05 DEBUG } 17:42:05 DEBUG ] 17:42:05 DEBUG } 17:42:05 DEBUG } 17:42:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544925" 17:42:05 INFO [http_cmd]: http status code OK 17:42:05 DEBUG --- status code --- 17:42:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:05 DEBUG --- http response --- 17:42:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:42:05 DEBUG ----- output ----- 17:42:05 DEBUG { 17:42:05 DEBUG "status": "success", 17:42:05 DEBUG "data": { 17:42:05 DEBUG "resultType": "vector", 17:42:05 DEBUG "result": [ 17:42:05 DEBUG { 17:42:05 DEBUG "metric": { 17:42:05 DEBUG "instance": "10.142.0.107:9100" 17:42:05 DEBUG }, 17:42:05 DEBUG "value": [ 17:42:05 DEBUG 17:42:05 DEBUG 17:42:05 DEBUG 17:42:05 DEBUG [...] 17:42:05 DEBUG 17:42:05 DEBUG 17:42:05 DEBUG 17:42:05 DEBUG "instance": "10.142.0.43:9100" 17:42:05 DEBUG }, 17:42:05 DEBUG "value": [ 17:42:05 DEBUG 1713544925, 17:42:05 DEBUG "1748117351424" 17:42:05 DEBUG ] 17:42:05 DEBUG } 17:42:05 DEBUG ] 17:42:05 DEBUG } 17:42:05 DEBUG } 17:42:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544926" 17:42:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544926" 17:42:06 INFO [http_cmd]: http status code OK 17:42:06 DEBUG --- status code --- 17:42:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:06 DEBUG --- http response --- 17:42:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:42:06 DEBUG ----- output ----- 17:42:06 DEBUG { 17:42:06 INFO [http_cmd]: http status code OK 17:42:06 DEBUG "status": "success", 17:42:06 DEBUG "data": { 17:42:06 DEBUG "resultType": "vector", 17:42:06 DEBUG "result": [ 17:42:06 DEBUG { 17:42:06 DEBUG "metric": { 17:42:06 DEBUG "pod": "ds-idrepo-0" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG [...] 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG "pod": "ds-idrepo-2" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 1713544926, 17:42:06 DEBUG "0" 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG } 17:42:06 DEBUG --- status code --- 17:42:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:06 DEBUG --- http response --- 17:42:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:42:06 DEBUG ----- output ----- 17:42:06 DEBUG { 17:42:06 DEBUG "status": "success", 17:42:06 DEBUG "data": { 17:42:06 DEBUG "resultType": "vector", 17:42:06 DEBUG "result": [ 17:42:06 DEBUG { 17:42:06 DEBUG "metric": { 17:42:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG [...] 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 1713544926, 17:42:06 DEBUG "1441058816" 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG } 17:42:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544926" 17:42:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544926" 17:42:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544926" 17:42:06 INFO [http_cmd]: http status code OK 17:42:06 DEBUG --- status code --- 17:42:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:06 DEBUG --- http response --- 17:42:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:42:06 DEBUG ----- output ----- 17:42:06 DEBUG { 17:42:06 DEBUG "status": "success", 17:42:06 DEBUG "data": { 17:42:06 DEBUG "resultType": "vector", 17:42:06 DEBUG "result": [ 17:42:06 DEBUG { 17:42:06 DEBUG "metric": { 17:42:06 DEBUG "pod": "ds-cts-0" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG [...] 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG "pod": "ds-cts-2" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 1713544926, 17:42:06 DEBUG "28.7" 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG } 17:42:06 INFO [http_cmd]: http status code OK 17:42:06 DEBUG --- status code --- 17:42:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:06 DEBUG --- http response --- 17:42:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:42:06 DEBUG ----- output ----- 17:42:06 DEBUG { 17:42:06 DEBUG "status": "success", 17:42:06 DEBUG "data": { 17:42:06 DEBUG "resultType": "vector", 17:42:06 DEBUG "result": [ 17:42:06 DEBUG { 17:42:06 DEBUG "metric": { 17:42:06 DEBUG "pod": "ds-cts-0" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG [...] 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG "pod": "ds-cts-2" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 1713544926, 17:42:06 DEBUG "0.19689999999999988" 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG } 17:42:06 INFO [http_cmd]: http status code OK 17:42:06 DEBUG --- status code --- 17:42:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:06 DEBUG --- http response --- 17:42:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:42:06 DEBUG ----- output ----- 17:42:06 DEBUG { 17:42:06 DEBUG "status": "success", 17:42:06 DEBUG "data": { 17:42:06 DEBUG "resultType": "vector", 17:42:06 DEBUG "result": [ 17:42:06 DEBUG { 17:42:06 DEBUG "metric": { 17:42:06 DEBUG "pod": "ds-idrepo-0" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG [...] 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG "pod": "ds-idrepo-2" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 1713544926, 17:42:06 DEBUG "0" 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG } 17:42:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544926" 17:42:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544926" 17:42:06 INFO [http_cmd]: http status code OK 17:42:06 DEBUG --- status code --- 17:42:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:06 DEBUG --- http response --- 17:42:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:42:06 DEBUG ----- output ----- 17:42:06 DEBUG { 17:42:06 DEBUG "status": "success", 17:42:06 DEBUG "data": { 17:42:06 DEBUG "resultType": "vector", 17:42:06 DEBUG "result": [ 17:42:06 DEBUG { 17:42:06 DEBUG "metric": { 17:42:06 DEBUG "instance": "10.142.0.107:9100" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG [...] 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG "instance": "10.142.0.43:9100" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 1713544926, 17:42:06 DEBUG "74800714.4" 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG } 17:42:06 INFO [http_cmd]: http status code OK 17:42:06 DEBUG --- status code --- 17:42:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:06 DEBUG --- http response --- 17:42:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:42:06 DEBUG ----- output ----- 17:42:06 DEBUG { 17:42:06 DEBUG "status": "success", 17:42:06 DEBUG "data": { 17:42:06 DEBUG "resultType": "vector", 17:42:06 DEBUG "result": [ 17:42:06 DEBUG { 17:42:06 DEBUG "metric": { 17:42:06 DEBUG "pod": "ds-idrepo-0" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG [...] 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 1713544926, 17:42:06 DEBUG "0" 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG } 17:42:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544926" 17:42:06 INFO [http_cmd]: http status code OK 17:42:06 DEBUG --- status code --- 17:42:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:06 DEBUG --- http response --- 17:42:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:42:06 DEBUG ----- output ----- 17:42:06 DEBUG { 17:42:06 DEBUG "status": "success", 17:42:06 DEBUG "data": { 17:42:06 DEBUG "resultType": "vector", 17:42:06 DEBUG "result": [ 17:42:06 DEBUG { 17:42:06 DEBUG "metric": { 17:42:06 DEBUG "instance": "10.142.0.107:9100" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG [...] 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG "instance": "10.142.0.43:9100" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 1713544926, 17:42:06 DEBUG "74795010.16666667" 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG } 17:42:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544926" 17:42:06 INFO [http_cmd]: http status code OK 17:42:06 DEBUG --- status code --- 17:42:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:06 DEBUG --- http response --- 17:42:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:42:06 DEBUG ----- output ----- 17:42:06 DEBUG { 17:42:06 DEBUG "status": "success", 17:42:06 DEBUG "data": { 17:42:06 DEBUG "resultType": "vector", 17:42:06 DEBUG "result": [ 17:42:06 DEBUG { 17:42:06 DEBUG "metric": { 17:42:06 DEBUG "pod": "ds-idrepo-0" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG [...] 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG 17:42:06 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:42:06 DEBUG }, 17:42:06 DEBUG "value": [ 17:42:06 DEBUG 1713544926, 17:42:06 DEBUG "0" 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG ] 17:42:06 DEBUG } 17:42:06 DEBUG } 17:42:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544927" 17:42:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544927" 17:42:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544927" 17:42:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544927" 17:42:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544927" 17:42:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544927" 17:42:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544927" 17:42:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544927" 17:42:07 INFO [http_cmd]: http status code OK 17:42:07 DEBUG --- status code --- 17:42:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:07 INFO [http_cmd]: http status code OK 17:42:07 DEBUG --- status code --- 17:42:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:07 DEBUG --- http response --- 17:42:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:42:07 DEBUG --- http response --- 17:42:07 INFO [http_cmd]: http status code OK 17:42:07 INFO [http_cmd]: http status code OK 17:42:07 DEBUG --- status code --- 17:42:07 INFO [http_cmd]: http status code OK 17:42:07 DEBUG ----- output ----- 17:42:07 INFO [http_cmd]: http status code OK 17:42:07 DEBUG --- status code --- 17:42:07 INFO [http_cmd]: http status code OK 17:42:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:42:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:07 DEBUG --- status code --- 17:42:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:07 DEBUG --- status code --- 17:42:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:07 DEBUG --- status code --- 17:42:07 DEBUG ----- output ----- 17:42:07 DEBUG --- http response --- 17:42:07 DEBUG --- http response --- 17:42:07 DEBUG { 17:42:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:07 DEBUG --- http response --- 17:42:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:07 DEBUG { 17:42:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:42:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:42:07 DEBUG "status": "success", 17:42:07 DEBUG --- http response --- 17:42:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:42:07 DEBUG --- http response --- 17:42:07 DEBUG "status": "success", 17:42:07 DEBUG ----- output ----- 17:42:07 DEBUG ----- output ----- 17:42:07 DEBUG "data": { 17:42:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:42:07 DEBUG ----- output ----- 17:42:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:42:07 DEBUG "data": { 17:42:07 DEBUG "resultType": "vector", 17:42:07 DEBUG "result": [ 17:42:07 DEBUG { 17:42:07 DEBUG "metric": { 17:42:07 DEBUG "pod": "ds-idrepo-0" 17:42:07 DEBUG ----- output ----- 17:42:07 DEBUG { 17:42:07 DEBUG { 17:42:07 DEBUG }, 17:42:07 DEBUG { 17:42:07 DEBUG ----- output ----- 17:42:07 DEBUG "resultType": "vector", 17:42:07 DEBUG { 17:42:07 DEBUG "status": "success", 17:42:07 DEBUG "data": { 17:42:07 DEBUG "value": [ 17:42:07 DEBUG "status": "success", 17:42:07 DEBUG { 17:42:07 DEBUG "result": [ 17:42:07 DEBUG "resultType": "vector", 17:42:07 DEBUG "status": "success", 17:42:07 DEBUG "status": "success", 17:42:07 DEBUG 17:42:07 DEBUG "data": { 17:42:07 DEBUG "resultType": "vector", 17:42:07 DEBUG { 17:42:07 DEBUG "result": [ 17:42:07 DEBUG "data": { 17:42:07 DEBUG "data": { 17:42:07 DEBUG "resultType": "vector", 17:42:07 DEBUG "status": "success", 17:42:07 DEBUG "data": { 17:42:07 DEBUG "metric": { 17:42:07 DEBUG { 17:42:07 DEBUG "resultType": "vector", 17:42:07 DEBUG "result": [ 17:42:07 DEBUG 17:42:07 DEBUG "result": [ 17:42:07 INFO [http_cmd]: http status code OK 17:42:07 DEBUG --- status code --- 17:42:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:42:07 DEBUG "metric": { 17:42:07 DEBUG "result": [ 17:42:07 DEBUG { 17:42:07 DEBUG 17:42:07 DEBUG [...] 17:42:07 DEBUG "resultType": "vector", 17:42:07 DEBUG "result": [ 17:42:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:42:07 DEBUG }, 17:42:07 DEBUG { 17:42:07 DEBUG "metric": { 17:42:07 DEBUG 17:42:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:42:07 DEBUG --- http response --- 17:42:07 DEBUG { 17:42:07 DEBUG { 17:42:07 DEBUG "value": [ 17:42:07 DEBUG "metric": { 17:42:07 DEBUG "pod": "ds-idrepo-0" 17:42:07 DEBUG 17:42:07 DEBUG }, 17:42:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:42:07 DEBUG "metric": { 17:42:07 DEBUG "metric": { 17:42:07 DEBUG "pod": "ds-cts-0" 17:42:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:42:07 DEBUG }, 17:42:07 DEBUG 17:42:07 DEBUG "value": [ 17:42:07 DEBUG ----- output ----- 17:42:07 DEBUG "instance": "10.142.0.107:9100" 17:42:07 DEBUG }, 17:42:07 DEBUG 17:42:07 DEBUG }, 17:42:07 DEBUG "value": [ 17:42:07 DEBUG "pod": "ds-idrepo-2" 17:42:07 DEBUG 17:42:07 DEBUG { 17:42:07 DEBUG }, 17:42:07 DEBUG "value": [ 17:42:07 DEBUG 17:42:07 DEBUG "value": [ 17:42:07 DEBUG 17:42:07 DEBUG }, 17:42:07 DEBUG 17:42:07 DEBUG "status": "success", 17:42:07 DEBUG "value": [ 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG "data": { 17:42:07 DEBUG "resultType": "vector", 17:42:07 DEBUG 17:42:07 DEBUG [...] 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG [...] 17:42:07 DEBUG [...] 17:42:07 DEBUG "result": [ 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG [...] 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG { 17:42:07 DEBUG 17:42:07 DEBUG [...] 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG "value": [ 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG "pod": "ds-idrepo-2" 17:42:07 DEBUG 1713544927, 17:42:07 DEBUG "metric": { 17:42:07 DEBUG }, 17:42:07 DEBUG "value": [ 17:42:07 DEBUG 1713544927, 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG 17:42:07 DEBUG "0" 17:42:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:42:07 DEBUG 17:42:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:42:07 DEBUG [...] 17:42:07 DEBUG }, 17:42:07 DEBUG "pod": "ds-cts-2" 17:42:07 DEBUG ] 17:42:07 DEBUG "2488.7" 17:42:07 DEBUG ] 17:42:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:42:07 DEBUG }, 17:42:07 DEBUG 17:42:07 DEBUG "value": [ 17:42:07 DEBUG }, 17:42:07 DEBUG } 17:42:07 DEBUG ] 17:42:07 DEBUG } 17:42:07 DEBUG }, 17:42:07 DEBUG "value": [ 17:42:07 DEBUG 17:42:07 DEBUG 1713544927, 17:42:07 DEBUG "value": [ 17:42:07 DEBUG 1713544927, 17:42:07 DEBUG "283.7" 17:42:07 DEBUG ] 17:42:07 DEBUG "value": [ 17:42:07 DEBUG 1713544927, 17:42:07 DEBUG 17:42:07 DEBUG "0" 17:42:07 DEBUG ] 17:42:07 DEBUG } 17:42:07 DEBUG } 17:42:07 DEBUG } 17:42:07 DEBUG } 17:42:07 DEBUG "0.023" 17:42:07 DEBUG "instance": "10.142.0.43:9100" 17:42:07 DEBUG ] 17:42:07 DEBUG } 17:42:07 DEBUG ] 17:42:07 DEBUG 1713544927, 17:42:07 DEBUG "2493.0666666666666" 17:42:07 DEBUG }, 17:42:07 DEBUG } 17:42:07 DEBUG }, 17:42:07 DEBUG } 17:42:07 DEBUG ] 17:42:07 DEBUG ] 17:42:07 DEBUG "value": [ 17:42:07 DEBUG } 17:42:07 DEBUG "value": [ 17:42:07 DEBUG ] 17:42:07 DEBUG } 17:42:07 DEBUG } 17:42:07 DEBUG 1713544927, 17:42:07 DEBUG 17:42:07 DEBUG } 17:42:07 DEBUG ] 17:42:07 DEBUG ] 17:42:07 DEBUG "0.9258666666667276" 17:42:07 DEBUG 17:42:07 DEBUG } 17:42:07 DEBUG } 17:42:07 DEBUG } 17:42:07 DEBUG ] 17:42:07 DEBUG 17:42:07 DEBUG } 17:42:07 DEBUG } 17:42:07 DEBUG [...] 17:42:07 DEBUG ] 17:42:07 DEBUG } 17:42:07 DEBUG 17:42:07 DEBUG } 17:42:07 DEBUG 17:42:07 DEBUG } 17:42:07 DEBUG 17:42:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:42:07 DEBUG }, 17:42:07 DEBUG "value": [ 17:42:07 DEBUG 1713544927, 17:42:07 DEBUG "8.038458257345031" 17:42:07 DEBUG ] 17:42:07 DEBUG } 17:42:07 DEBUG ] 17:42:07 DEBUG } 17:42:07 DEBUG } 17:42:07 INFO 17:42:07 INFO [loop_until]: kubectl --namespace=xlou top pods 17:42:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:42:07 INFO [loop_until]: OK (rc = 0) 17:42:07 DEBUG --- stdout --- 17:42:07 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17107m 9001Mi am-6fd554f8f7-btd4s 17309m 9039Mi am-6fd554f8f7-wxcsj 18393m 9027Mi ds-cts-0 21250m 12923Mi ds-cts-1 21814m 12946Mi ds-cts-2 22188m 13044Mi ds-idrepo-0 7016m 3422Mi ds-idrepo-1 6987m 3392Mi ds-idrepo-2 7254m 3461Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 3m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8006m 1280Mi 17:42:07 DEBUG --- stderr --- 17:42:07 DEBUG 17:42:07 INFO 17:42:07 INFO [loop_until]: kubectl --namespace=xlou top node 17:42:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:42:08 INFO [loop_until]: OK (rc = 0) 17:42:08 DEBUG --- stdout --- 17:42:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 59m 0% 6173Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7471m 25% 6995Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 46m 0% 2677Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 16929m 56% 12300Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 21273m 71% 15969Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 145m 0% 5250Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17465m 58% 11935Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16820m 56% 3027Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19003m 63% 11263Mi 10% gke-xlou-cdm-ds-0f601751-j488 7602m 25% 6689Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21606m 72% 17645Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7436m 24% 7226Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21696m 72% 15490Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8174m 51% 5025Mi 8% 17:42:08 DEBUG --- stderr --- 17:42:08 DEBUG 17:43:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713544984" 17:43:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544984" 17:43:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544984" 17:43:04 ERROR [http_cmd]: ERROR 17:43:04 INFO [http_cmd]: http status code OK 17:43:04 ERROR --- status code --- 17:43:04 DEBUG --- status code --- 17:43:04 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:43:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:04 ERROR --- http response --- 17:43:04 DEBUG --- http response --- 17:43:04 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:43:04 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:43:04 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:43:04 INFO [http_cmd]: http status code OK 17:43:04 DEBUG --- status code --- 17:43:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:04 DEBUG --- http response --- 17:43:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:43:04 DEBUG ----- output ----- 17:43:04 DEBUG { 17:43:04 DEBUG "status": "success", 17:43:04 DEBUG "data": { 17:43:04 DEBUG "resultType": "vector", 17:43:04 DEBUG "result": [ 17:43:04 DEBUG { 17:43:04 DEBUG "metric": { 17:43:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:43:04 DEBUG }, 17:43:04 DEBUG "value": [ 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG [...] 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:43:04 DEBUG }, 17:43:04 DEBUG "value": [ 17:43:04 DEBUG 1713544984, 17:43:04 DEBUG "2317.2" 17:43:04 DEBUG ] 17:43:04 DEBUG } 17:43:04 DEBUG ] 17:43:04 DEBUG } 17:43:04 DEBUG } 17:43:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713544984" 17:43:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544984" 17:43:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544984" 17:43:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544984" 17:43:04 INFO [http_cmd]: http status code OK 17:43:04 DEBUG --- status code --- 17:43:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:04 DEBUG --- http response --- 17:43:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:43:04 DEBUG ----- output ----- 17:43:04 DEBUG { 17:43:04 DEBUG "status": "success", 17:43:04 DEBUG "data": { 17:43:04 DEBUG "resultType": "vector", 17:43:04 DEBUG "result": [ 17:43:04 DEBUG { 17:43:04 DEBUG "metric": { 17:43:04 DEBUG "pod": "ds-idrepo-0" 17:43:04 DEBUG }, 17:43:04 DEBUG "value": [ 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG [...] 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG "pod": "ds-cts-2" 17:43:04 DEBUG }, 17:43:04 DEBUG "value": [ 17:43:04 DEBUG 1713544984, 17:43:04 DEBUG "0" 17:43:04 DEBUG ] 17:43:04 DEBUG } 17:43:04 INFO [http_cmd]: http status code OK 17:43:04 DEBUG ] 17:43:04 DEBUG --- status code --- 17:43:04 DEBUG } 17:43:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:04 DEBUG } 17:43:04 DEBUG --- http response --- 17:43:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:43:04 DEBUG ----- output ----- 17:43:04 DEBUG { 17:43:04 DEBUG "status": "success", 17:43:04 DEBUG "data": { 17:43:04 DEBUG "resultType": "vector", 17:43:04 DEBUG "result": [ 17:43:04 DEBUG { 17:43:04 DEBUG "metric": { 17:43:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:43:04 DEBUG }, 17:43:04 DEBUG "value": [ 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG [...] 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:43:04 DEBUG }, 17:43:04 DEBUG "value": [ 17:43:04 DEBUG 1713544984, 17:43:04 DEBUG "0" 17:43:04 DEBUG ] 17:43:04 DEBUG } 17:43:04 DEBUG ] 17:43:04 DEBUG } 17:43:04 DEBUG } 17:43:04 INFO [http_cmd]: http status code OK 17:43:04 DEBUG --- status code --- 17:43:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:04 DEBUG --- http response --- 17:43:04 INFO [http_cmd]: http status code OK 17:43:04 DEBUG --- status code --- 17:43:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:43:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:04 DEBUG ----- output ----- 17:43:04 DEBUG --- http response --- 17:43:04 DEBUG { 17:43:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:43:04 DEBUG "status": "success", 17:43:04 DEBUG ----- output ----- 17:43:04 DEBUG "data": { 17:43:04 DEBUG { 17:43:04 DEBUG "resultType": "vector", 17:43:04 DEBUG "status": "success", 17:43:04 DEBUG "result": [ 17:43:04 DEBUG "data": { 17:43:04 DEBUG { 17:43:04 DEBUG "resultType": "vector", 17:43:04 DEBUG "metric": { 17:43:04 DEBUG "result": [ 17:43:04 DEBUG { 17:43:04 DEBUG "metric": { 17:43:04 DEBUG "pod": "ds-idrepo-1" 17:43:04 DEBUG }, 17:43:04 DEBUG "value": [ 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG [...] 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:43:04 DEBUG }, 17:43:04 DEBUG "value": [ 17:43:04 DEBUG 1713544984, 17:43:04 DEBUG "0" 17:43:04 DEBUG ] 17:43:04 DEBUG } 17:43:04 DEBUG ] 17:43:04 DEBUG } 17:43:04 DEBUG } 17:43:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:43:04 DEBUG }, 17:43:04 DEBUG "value": [ 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG [...] 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:43:04 DEBUG }, 17:43:04 DEBUG "value": [ 17:43:04 DEBUG 1713544984, 17:43:04 DEBUG "2492.4" 17:43:04 DEBUG ] 17:43:04 DEBUG } 17:43:04 DEBUG ] 17:43:04 DEBUG } 17:43:04 DEBUG } 17:43:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544984" 17:43:04 INFO [http_cmd]: http status code OK 17:43:04 DEBUG --- status code --- 17:43:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:04 DEBUG --- http response --- 17:43:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:43:04 DEBUG ----- output ----- 17:43:04 DEBUG { 17:43:04 DEBUG "status": "success", 17:43:04 DEBUG "data": { 17:43:04 DEBUG "resultType": "vector", 17:43:04 DEBUG "result": [ 17:43:04 DEBUG { 17:43:04 DEBUG "metric": { 17:43:04 DEBUG "instance": "10.142.0.107:9100" 17:43:04 DEBUG }, 17:43:04 DEBUG "value": [ 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG [...] 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG 17:43:04 DEBUG "instance": "10.142.0.43:9100" 17:43:04 DEBUG }, 17:43:04 DEBUG "value": [ 17:43:04 DEBUG 1713544984, 17:43:04 DEBUG "0.472333333333332" 17:43:04 DEBUG ] 17:43:04 DEBUG } 17:43:04 DEBUG ] 17:43:04 DEBUG } 17:43:04 DEBUG } 17:43:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544985" 17:43:05 INFO [http_cmd]: http status code OK 17:43:05 DEBUG --- status code --- 17:43:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:05 DEBUG --- http response --- 17:43:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:43:05 DEBUG ----- output ----- 17:43:05 DEBUG { 17:43:05 DEBUG "status": "success", 17:43:05 DEBUG "data": { 17:43:05 DEBUG "resultType": "vector", 17:43:05 DEBUG "result": [ 17:43:05 DEBUG { 17:43:05 DEBUG "metric": { 17:43:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:43:05 DEBUG }, 17:43:05 DEBUG "value": [ 17:43:05 DEBUG 17:43:05 DEBUG 17:43:05 DEBUG 17:43:05 DEBUG [...] 17:43:05 DEBUG 17:43:05 DEBUG 17:43:05 DEBUG 17:43:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:43:05 DEBUG }, 17:43:05 DEBUG "value": [ 17:43:05 DEBUG 1713544985, 17:43:05 DEBUG "4175908.5" 17:43:05 DEBUG ] 17:43:05 DEBUG } 17:43:05 DEBUG ] 17:43:05 DEBUG } 17:43:05 DEBUG } 17:43:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544985" 17:43:05 INFO [http_cmd]: http status code OK 17:43:05 DEBUG --- status code --- 17:43:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:05 DEBUG --- http response --- 17:43:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:43:05 DEBUG ----- output ----- 17:43:05 DEBUG { 17:43:05 DEBUG "status": "success", 17:43:05 DEBUG "data": { 17:43:05 DEBUG "resultType": "vector", 17:43:05 DEBUG "result": [ 17:43:05 DEBUG { 17:43:05 DEBUG "metric": { 17:43:05 DEBUG "instance": "10.142.0.107:9100" 17:43:05 DEBUG }, 17:43:05 DEBUG "value": [ 17:43:05 DEBUG 17:43:05 DEBUG 17:43:05 DEBUG 17:43:05 DEBUG [...] 17:43:05 DEBUG 17:43:05 DEBUG 17:43:05 DEBUG 17:43:05 DEBUG "instance": "10.142.0.43:9100" 17:43:05 DEBUG }, 17:43:05 DEBUG "value": [ 17:43:05 DEBUG 1713544985, 17:43:05 DEBUG "1751985392640" 17:43:05 DEBUG ] 17:43:05 DEBUG } 17:43:05 DEBUG ] 17:43:05 DEBUG } 17:43:05 DEBUG } 17:43:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544986" 17:43:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713544986" 17:43:06 INFO [http_cmd]: http status code OK 17:43:06 DEBUG --- status code --- 17:43:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:06 DEBUG --- http response --- 17:43:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:43:06 DEBUG ----- output ----- 17:43:06 DEBUG { 17:43:06 DEBUG "status": "success", 17:43:06 DEBUG "data": { 17:43:06 DEBUG "resultType": "vector", 17:43:06 DEBUG "result": [ 17:43:06 DEBUG { 17:43:06 DEBUG "metric": { 17:43:06 DEBUG "pod": "ds-idrepo-0" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG [...] 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG "pod": "ds-idrepo-2" 17:43:06 INFO [http_cmd]: http status code OK 17:43:06 DEBUG }, 17:43:06 DEBUG --- status code --- 17:43:06 DEBUG "value": [ 17:43:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:06 DEBUG 1713544986, 17:43:06 DEBUG "0" 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG } 17:43:06 DEBUG --- http response --- 17:43:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:43:06 DEBUG ----- output ----- 17:43:06 DEBUG { 17:43:06 DEBUG "status": "success", 17:43:06 DEBUG "data": { 17:43:06 DEBUG "resultType": "vector", 17:43:06 DEBUG "result": [ 17:43:06 DEBUG { 17:43:06 DEBUG "metric": { 17:43:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG [...] 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 1713544986, 17:43:06 DEBUG "1440796672" 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG } 17:43:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544986" 17:43:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544986" 17:43:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544986" 17:43:06 INFO [http_cmd]: http status code OK 17:43:06 DEBUG --- status code --- 17:43:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:06 DEBUG --- http response --- 17:43:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:43:06 DEBUG ----- output ----- 17:43:06 DEBUG { 17:43:06 DEBUG "status": "success", 17:43:06 DEBUG "data": { 17:43:06 DEBUG "resultType": "vector", 17:43:06 DEBUG "result": [ 17:43:06 DEBUG { 17:43:06 DEBUG "metric": { 17:43:06 DEBUG "pod": "ds-cts-0" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG [...] 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG "pod": "ds-cts-2" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 1713544986, 17:43:06 DEBUG "30.599999999999998" 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG } 17:43:06 INFO [http_cmd]: http status code OK 17:43:06 DEBUG --- status code --- 17:43:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:06 DEBUG --- http response --- 17:43:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:43:06 DEBUG ----- output ----- 17:43:06 DEBUG { 17:43:06 DEBUG "status": "success", 17:43:06 DEBUG "data": { 17:43:06 DEBUG "resultType": "vector", 17:43:06 INFO [http_cmd]: http status code OK 17:43:06 DEBUG --- status code --- 17:43:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:06 DEBUG --- http response --- 17:43:06 DEBUG "result": [ 17:43:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:43:06 DEBUG { 17:43:06 DEBUG ----- output ----- 17:43:06 DEBUG "metric": { 17:43:06 DEBUG { 17:43:06 DEBUG "status": "success", 17:43:06 DEBUG "pod": "ds-cts-0" 17:43:06 DEBUG "data": { 17:43:06 DEBUG }, 17:43:06 DEBUG "resultType": "vector", 17:43:06 DEBUG "result": [ 17:43:06 DEBUG { 17:43:06 DEBUG "metric": { 17:43:06 DEBUG "value": [ 17:43:06 DEBUG "pod": "ds-idrepo-0" 17:43:06 DEBUG 17:43:06 DEBUG }, 17:43:06 DEBUG 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG [...] 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG [...] 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG "pod": "ds-idrepo-2" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 1713544986, 17:43:06 DEBUG "0" 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG } 17:43:06 DEBUG "pod": "ds-cts-2" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 1713544986, 17:43:06 DEBUG "0.21873333333333278" 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG } 17:43:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544986" 17:43:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544986" 17:43:06 INFO [http_cmd]: http status code OK 17:43:06 INFO [http_cmd]: http status code OK 17:43:06 DEBUG --- status code --- 17:43:06 DEBUG --- status code --- 17:43:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:06 DEBUG --- http response --- 17:43:06 DEBUG --- http response --- 17:43:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:43:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:43:06 DEBUG ----- output ----- 17:43:06 DEBUG ----- output ----- 17:43:06 DEBUG { 17:43:06 DEBUG { 17:43:06 DEBUG "status": "success", 17:43:06 DEBUG "status": "success", 17:43:06 DEBUG "data": { 17:43:06 DEBUG "resultType": "vector", 17:43:06 DEBUG "data": { 17:43:06 DEBUG "resultType": "vector", 17:43:06 DEBUG "result": [ 17:43:06 DEBUG { 17:43:06 DEBUG "metric": { 17:43:06 DEBUG "pod": "ds-idrepo-1" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG [...] 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG "pod": "ds-idrepo-2" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 1713544986, 17:43:06 DEBUG "4425.353588266108" 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG } 17:43:06 DEBUG "result": [ 17:43:06 DEBUG { 17:43:06 DEBUG "metric": { 17:43:06 DEBUG "instance": "10.142.0.107:9100" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG [...] 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG "instance": "10.142.0.43:9100" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 1713544986, 17:43:06 DEBUG "75025651.83333334" 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG } 17:43:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544986" 17:43:06 INFO [http_cmd]: http status code OK 17:43:06 DEBUG --- status code --- 17:43:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:06 DEBUG --- http response --- 17:43:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:43:06 DEBUG ----- output ----- 17:43:06 DEBUG { 17:43:06 DEBUG "status": "success", 17:43:06 DEBUG "data": { 17:43:06 DEBUG "resultType": "vector", 17:43:06 DEBUG "result": [ 17:43:06 DEBUG { 17:43:06 DEBUG "metric": { 17:43:06 DEBUG "instance": "10.142.0.107:9100" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG [...] 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG "instance": "10.142.0.43:9100" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 1713544986, 17:43:06 DEBUG "75018939.9" 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG } 17:43:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544986" 17:43:06 INFO [http_cmd]: http status code OK 17:43:06 DEBUG --- status code --- 17:43:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:06 DEBUG --- http response --- 17:43:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:43:06 DEBUG ----- output ----- 17:43:06 DEBUG { 17:43:06 DEBUG "status": "success", 17:43:06 DEBUG "data": { 17:43:06 DEBUG "resultType": "vector", 17:43:06 DEBUG "result": [ 17:43:06 DEBUG { 17:43:06 DEBUG "metric": { 17:43:06 DEBUG "pod": "ds-idrepo-1" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG [...] 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG 17:43:06 DEBUG "pod": "ds-idrepo-2" 17:43:06 DEBUG }, 17:43:06 DEBUG "value": [ 17:43:06 DEBUG 1713544986, 17:43:06 DEBUG "0" 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG ] 17:43:06 DEBUG } 17:43:06 DEBUG } 17:43:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544987" 17:43:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544987" 17:43:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544987" 17:43:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544987" 17:43:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713544987" 17:43:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713544987" 17:43:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713544987" 17:43:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713544987" 17:43:07 INFO [http_cmd]: http status code OK 17:43:07 INFO [http_cmd]: http status code OK 17:43:07 DEBUG --- status code --- 17:43:07 INFO [http_cmd]: http status code OK 17:43:07 INFO [http_cmd]: http status code OK 17:43:07 DEBUG --- status code --- 17:43:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:07 DEBUG --- status code --- 17:43:07 DEBUG --- status code --- 17:43:07 INFO [http_cmd]: http status code OK 17:43:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:07 INFO [http_cmd]: http status code OK 17:43:07 DEBUG --- http response --- 17:43:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:07 DEBUG --- status code --- 17:43:07 DEBUG --- http response --- 17:43:07 DEBUG --- status code --- 17:43:07 DEBUG --- http response --- 17:43:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:43:07 DEBUG --- http response --- 17:43:07 INFO [http_cmd]: http status code OK 17:43:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:43:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:43:07 DEBUG ----- output ----- 17:43:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:43:07 DEBUG --- status code --- 17:43:07 DEBUG --- http response --- 17:43:07 DEBUG ----- output ----- 17:43:07 INFO [http_cmd]: http status code OK 17:43:07 DEBUG --- http response --- 17:43:07 DEBUG { 17:43:07 DEBUG ----- output ----- 17:43:07 DEBUG { 17:43:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:43:07 DEBUG { 17:43:07 DEBUG --- status code --- 17:43:07 DEBUG "status": "success", 17:43:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:43:07 DEBUG ----- output ----- 17:43:07 DEBUG "status": "success", 17:43:07 DEBUG --- http response --- 17:43:07 DEBUG ----- output ----- 17:43:07 DEBUG "status": "success", 17:43:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:43:07 DEBUG "data": { 17:43:07 DEBUG ----- output ----- 17:43:07 DEBUG { 17:43:07 DEBUG "data": { 17:43:07 DEBUG { 17:43:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:43:07 DEBUG "data": { 17:43:07 DEBUG --- http response --- 17:43:07 DEBUG "resultType": "vector", 17:43:07 DEBUG { 17:43:07 DEBUG "status": "success", 17:43:07 DEBUG "resultType": "vector", 17:43:07 DEBUG "status": "success", 17:43:07 DEBUG ----- output ----- 17:43:07 DEBUG "resultType": "vector", 17:43:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:43:07 DEBUG "result": [ 17:43:07 DEBUG "status": "success", 17:43:07 DEBUG "data": { 17:43:07 DEBUG "result": [ 17:43:07 DEBUG "data": { 17:43:07 DEBUG { 17:43:07 DEBUG "result": [ 17:43:07 DEBUG ----- output ----- 17:43:07 DEBUG { 17:43:07 DEBUG "data": { 17:43:07 DEBUG "resultType": "vector", 17:43:07 DEBUG { 17:43:07 DEBUG "resultType": "vector", 17:43:07 DEBUG "status": "success", 17:43:07 DEBUG { 17:43:07 DEBUG { 17:43:07 DEBUG "status": "success", 17:43:07 DEBUG "resultType": "vector", 17:43:07 DEBUG "result": [ 17:43:07 DEBUG "metric": { 17:43:07 DEBUG "result": [ 17:43:07 DEBUG "data": { 17:43:07 DEBUG "metric": { 17:43:07 DEBUG "metric": { 17:43:07 DEBUG "data": { 17:43:07 DEBUG "resultType": "vector", 17:43:07 DEBUG { 17:43:07 DEBUG "pod": "ds-idrepo-0" 17:43:07 DEBUG { 17:43:07 DEBUG "resultType": "vector", 17:43:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:43:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:43:07 DEBUG "result": [ 17:43:07 DEBUG { 17:43:07 DEBUG "metric": { 17:43:07 DEBUG }, 17:43:07 DEBUG "metric": { 17:43:07 DEBUG "result": [ 17:43:07 DEBUG }, 17:43:07 DEBUG }, 17:43:07 DEBUG "result": [ 17:43:07 DEBUG "metric": { 17:43:07 DEBUG "pod": "ds-cts-0" 17:43:07 DEBUG "value": [ 17:43:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:43:07 DEBUG { 17:43:07 DEBUG "value": [ 17:43:07 DEBUG "value": [ 17:43:07 DEBUG { 17:43:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:43:07 DEBUG }, 17:43:07 DEBUG 17:43:07 DEBUG }, 17:43:07 DEBUG "metric": { 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG "metric": { 17:43:07 DEBUG }, 17:43:07 DEBUG "value": [ 17:43:07 DEBUG 17:43:07 DEBUG "value": [ 17:43:07 DEBUG "instance": "10.142.0.107:9100" 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:43:07 DEBUG "value": [ 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG }, 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG }, 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG [...] 17:43:07 DEBUG 17:43:07 DEBUG "value": [ 17:43:07 DEBUG [...] 17:43:07 DEBUG [...] 17:43:07 DEBUG "value": [ 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG [...] 17:43:07 DEBUG 17:43:07 DEBUG [...] 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG [...] 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG "pod": "ds-idrepo-2" 17:43:07 DEBUG 17:43:07 DEBUG [...] 17:43:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:43:07 DEBUG "pod": "ds-idrepo-2" 17:43:07 DEBUG [...] 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG }, 17:43:07 DEBUG "value": [ 17:43:07 DEBUG 17:43:07 DEBUG }, 17:43:07 DEBUG }, 17:43:07 DEBUG 17:43:07 DEBUG 17:43:07 DEBUG "pod": "ds-cts-2" 17:43:07 DEBUG 17:43:07 DEBUG 1713544987, 17:43:07 DEBUG 17:43:07 DEBUG "value": [ 17:43:07 DEBUG "value": [ 17:43:07 DEBUG 17:43:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:43:07 DEBUG }, 17:43:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:43:07 DEBUG "0" 17:43:07 DEBUG 17:43:07 DEBUG 1713544987, 17:43:07 DEBUG 1713544987, 17:43:07 DEBUG 17:43:07 DEBUG }, 17:43:07 DEBUG "value": [ 17:43:07 DEBUG }, 17:43:07 DEBUG ] 17:43:07 DEBUG "instance": "10.142.0.43:9100" 17:43:07 DEBUG "2499.6" 17:43:07 DEBUG "7.080512015452492" 17:43:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:43:07 DEBUG "value": [ 17:43:07 DEBUG 1713544987, 17:43:07 DEBUG "value": [ 17:43:07 DEBUG } 17:43:07 DEBUG }, 17:43:07 DEBUG ] 17:43:07 DEBUG ] 17:43:07 DEBUG }, 17:43:07 DEBUG 1713544987, 17:43:07 DEBUG "317" 17:43:07 DEBUG 1713544987, 17:43:07 DEBUG ] 17:43:07 DEBUG "value": [ 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 DEBUG "value": [ 17:43:07 DEBUG "8.040384457702341" 17:43:07 DEBUG ] 17:43:07 DEBUG "2490.866666666667" 17:43:07 DEBUG } 17:43:07 DEBUG 1713544987, 17:43:07 DEBUG ] 17:43:07 DEBUG ] 17:43:07 DEBUG 1713544987, 17:43:07 DEBUG ] 17:43:07 DEBUG } 17:43:07 DEBUG ] 17:43:07 DEBUG } 17:43:07 DEBUG "0.8536666666666178" 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 DEBUG "0.023" 17:43:07 DEBUG ] 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 DEBUG ] 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 DEBUG ] 17:43:07 DEBUG ] 17:43:07 DEBUG ] 17:43:07 DEBUG } 17:43:07 DEBUG ] 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 DEBUG ] 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 DEBUG } 17:43:07 INFO 17:43:07 INFO [loop_until]: kubectl --namespace=xlou top pods 17:43:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:43:07 INFO [loop_until]: OK (rc = 0) 17:43:07 DEBUG --- stdout --- 17:43:07 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17162m 9002Mi am-6fd554f8f7-btd4s 17001m 9038Mi am-6fd554f8f7-wxcsj 19199m 9026Mi ds-cts-0 20992m 12878Mi ds-cts-1 21597m 12758Mi ds-cts-2 21888m 12779Mi ds-idrepo-0 7066m 3423Mi ds-idrepo-1 7005m 3394Mi ds-idrepo-2 7192m 3462Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8089m 1287Mi 17:43:07 DEBUG --- stderr --- 17:43:07 DEBUG 17:43:08 INFO 17:43:08 INFO [loop_until]: kubectl --namespace=xlou top node 17:43:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:43:08 INFO [loop_until]: OK (rc = 0) 17:43:08 DEBUG --- stdout --- 17:43:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6174Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7509m 25% 6998Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 49m 0% 2684Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17520m 58% 12303Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 21153m 70% 15870Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5237Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17426m 58% 11937Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16778m 56% 3028Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19268m 64% 11264Mi 10% gke-xlou-cdm-ds-0f601751-j488 7650m 25% 6689Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21273m 71% 17548Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7488m 25% 7236Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21757m 72% 15208Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8187m 51% 5028Mi 8% 17:43:08 DEBUG --- stderr --- 17:43:08 DEBUG 17:44:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545044" 17:44:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545044" 17:44:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545044" 17:44:04 ERROR [http_cmd]: ERROR 17:44:04 INFO [http_cmd]: http status code OK 17:44:04 ERROR --- status code --- 17:44:04 DEBUG --- status code --- 17:44:04 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:44:04 ERROR --- http response --- 17:44:04 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:44:04 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:44:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:04 DEBUG --- http response --- 17:44:04 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:44:04 INFO [http_cmd]: http status code OK 17:44:04 DEBUG --- status code --- 17:44:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:04 DEBUG --- http response --- 17:44:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:44:04 DEBUG ----- output ----- 17:44:04 DEBUG { 17:44:04 DEBUG "status": "success", 17:44:04 DEBUG "data": { 17:44:04 DEBUG "resultType": "vector", 17:44:04 DEBUG "result": [ 17:44:04 DEBUG { 17:44:04 DEBUG "metric": { 17:44:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:44:04 DEBUG }, 17:44:04 DEBUG "value": [ 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG [...] 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:44:04 DEBUG }, 17:44:04 DEBUG "value": [ 17:44:04 DEBUG 1713545044, 17:44:04 DEBUG "2331.3333333333335" 17:44:04 DEBUG ] 17:44:04 DEBUG } 17:44:04 DEBUG ] 17:44:04 DEBUG } 17:44:04 DEBUG } 17:44:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545044" 17:44:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545044" 17:44:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545044" 17:44:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545044" 17:44:04 INFO [http_cmd]: http status code OK 17:44:04 DEBUG --- status code --- 17:44:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:04 DEBUG --- http response --- 17:44:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:44:04 DEBUG ----- output ----- 17:44:04 DEBUG { 17:44:04 DEBUG "status": "success", 17:44:04 DEBUG "data": { 17:44:04 DEBUG "resultType": "vector", 17:44:04 DEBUG "result": [ 17:44:04 DEBUG { 17:44:04 DEBUG "metric": { 17:44:04 DEBUG "pod": "ds-idrepo-0" 17:44:04 DEBUG }, 17:44:04 DEBUG "value": [ 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG [...] 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG "pod": "ds-cts-2" 17:44:04 DEBUG }, 17:44:04 DEBUG "value": [ 17:44:04 DEBUG 1713545044, 17:44:04 DEBUG "0" 17:44:04 DEBUG ] 17:44:04 DEBUG } 17:44:04 DEBUG ] 17:44:04 DEBUG } 17:44:04 DEBUG } 17:44:04 INFO [http_cmd]: http status code OK 17:44:04 DEBUG --- status code --- 17:44:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:04 DEBUG --- http response --- 17:44:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:44:04 DEBUG ----- output ----- 17:44:04 DEBUG { 17:44:04 DEBUG "status": "success", 17:44:04 DEBUG "data": { 17:44:04 DEBUG "resultType": "vector", 17:44:04 DEBUG "result": [ 17:44:04 DEBUG { 17:44:04 DEBUG "metric": { 17:44:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:44:04 DEBUG }, 17:44:04 DEBUG "value": [ 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG [...] 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:44:04 DEBUG }, 17:44:04 DEBUG "value": [ 17:44:04 DEBUG 1713545044, 17:44:04 DEBUG "0" 17:44:04 DEBUG ] 17:44:04 DEBUG } 17:44:04 DEBUG ] 17:44:04 DEBUG } 17:44:04 DEBUG } 17:44:04 INFO [http_cmd]: http status code OK 17:44:04 DEBUG --- status code --- 17:44:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:04 DEBUG --- http response --- 17:44:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:44:04 DEBUG ----- output ----- 17:44:04 DEBUG { 17:44:04 DEBUG "status": "success", 17:44:04 DEBUG "data": { 17:44:04 DEBUG "resultType": "vector", 17:44:04 DEBUG "result": [ 17:44:04 DEBUG { 17:44:04 DEBUG "metric": { 17:44:04 DEBUG "pod": "ds-idrepo-1" 17:44:04 DEBUG }, 17:44:04 DEBUG "value": [ 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG [...] 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG "pod": "ds-idrepo-2" 17:44:04 DEBUG }, 17:44:04 DEBUG "value": [ 17:44:04 DEBUG 1713545044, 17:44:04 DEBUG "146870407.90439948" 17:44:04 DEBUG ] 17:44:04 DEBUG } 17:44:04 INFO [http_cmd]: http status code OK 17:44:04 DEBUG ] 17:44:04 DEBUG --- status code --- 17:44:04 DEBUG } 17:44:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:04 DEBUG } 17:44:04 DEBUG --- http response --- 17:44:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:44:04 DEBUG ----- output ----- 17:44:04 DEBUG { 17:44:04 DEBUG "status": "success", 17:44:04 DEBUG "data": { 17:44:04 DEBUG "resultType": "vector", 17:44:04 DEBUG "result": [ 17:44:04 DEBUG { 17:44:04 DEBUG "metric": { 17:44:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:44:04 DEBUG }, 17:44:04 DEBUG "value": [ 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG [...] 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG 17:44:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:44:04 DEBUG }, 17:44:04 DEBUG "value": [ 17:44:04 DEBUG 1713545044, 17:44:04 DEBUG "2504.366666666667" 17:44:04 DEBUG ] 17:44:04 DEBUG } 17:44:04 DEBUG ] 17:44:04 DEBUG } 17:44:04 DEBUG } 17:44:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545044" 17:44:05 INFO [http_cmd]: http status code OK 17:44:05 DEBUG --- status code --- 17:44:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:05 DEBUG --- http response --- 17:44:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:44:05 DEBUG ----- output ----- 17:44:05 DEBUG { 17:44:05 DEBUG "status": "success", 17:44:05 DEBUG "data": { 17:44:05 DEBUG "resultType": "vector", 17:44:05 DEBUG "result": [ 17:44:05 DEBUG { 17:44:05 DEBUG "metric": { 17:44:05 DEBUG "instance": "10.142.0.107:9100" 17:44:05 DEBUG }, 17:44:05 DEBUG "value": [ 17:44:05 DEBUG 17:44:05 DEBUG 17:44:05 DEBUG 17:44:05 DEBUG [...] 17:44:05 DEBUG 17:44:05 DEBUG 17:44:05 DEBUG 17:44:05 DEBUG "instance": "10.142.0.43:9100" 17:44:05 DEBUG }, 17:44:05 DEBUG "value": [ 17:44:05 DEBUG 1713545044, 17:44:05 DEBUG "0.4923333333333346" 17:44:05 DEBUG ] 17:44:05 DEBUG } 17:44:05 DEBUG ] 17:44:05 DEBUG } 17:44:05 DEBUG } 17:44:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545045" 17:44:05 INFO [http_cmd]: http status code OK 17:44:05 DEBUG --- status code --- 17:44:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:05 DEBUG --- http response --- 17:44:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:44:05 DEBUG ----- output ----- 17:44:05 DEBUG { 17:44:05 DEBUG "status": "success", 17:44:05 DEBUG "data": { 17:44:05 DEBUG "resultType": "vector", 17:44:05 DEBUG "result": [ 17:44:05 DEBUG { 17:44:05 DEBUG "metric": { 17:44:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:44:05 DEBUG }, 17:44:05 DEBUG "value": [ 17:44:05 DEBUG 17:44:05 DEBUG 17:44:05 DEBUG 17:44:05 DEBUG [...] 17:44:05 DEBUG 17:44:05 DEBUG 17:44:05 DEBUG 17:44:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:44:05 DEBUG }, 17:44:05 DEBUG "value": [ 17:44:05 DEBUG 1713545045, 17:44:05 DEBUG "4185993.5" 17:44:05 DEBUG ] 17:44:05 DEBUG } 17:44:05 DEBUG ] 17:44:05 DEBUG } 17:44:05 DEBUG } 17:44:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545045" 17:44:05 INFO [http_cmd]: http status code OK 17:44:05 DEBUG --- status code --- 17:44:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:05 DEBUG --- http response --- 17:44:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:44:05 DEBUG ----- output ----- 17:44:05 DEBUG { 17:44:05 DEBUG "status": "success", 17:44:05 DEBUG "data": { 17:44:05 DEBUG "resultType": "vector", 17:44:05 DEBUG "result": [ 17:44:05 DEBUG { 17:44:05 DEBUG "metric": { 17:44:05 DEBUG "instance": "10.142.0.107:9100" 17:44:05 DEBUG }, 17:44:05 DEBUG "value": [ 17:44:05 DEBUG 17:44:05 DEBUG 17:44:05 DEBUG 17:44:05 DEBUG [...] 17:44:05 DEBUG 17:44:05 DEBUG 17:44:05 DEBUG 17:44:05 DEBUG "instance": "10.142.0.43:9100" 17:44:05 DEBUG }, 17:44:05 DEBUG "value": [ 17:44:05 DEBUG 1713545045, 17:44:05 DEBUG "1756202681344" 17:44:05 DEBUG ] 17:44:05 DEBUG } 17:44:05 DEBUG ] 17:44:05 DEBUG } 17:44:05 DEBUG } 17:44:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545046" 17:44:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545046" 17:44:06 INFO [http_cmd]: http status code OK 17:44:06 DEBUG --- status code --- 17:44:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:06 DEBUG --- http response --- 17:44:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:44:06 DEBUG ----- output ----- 17:44:06 DEBUG { 17:44:06 DEBUG "status": "success", 17:44:06 DEBUG "data": { 17:44:06 DEBUG "resultType": "vector", 17:44:06 DEBUG "result": [ 17:44:06 DEBUG { 17:44:06 DEBUG "metric": { 17:44:06 DEBUG "pod": "ds-idrepo-0" 17:44:06 DEBUG }, 17:44:06 DEBUG "value": [ 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG [...] 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG "pod": "ds-idrepo-2" 17:44:06 DEBUG }, 17:44:06 DEBUG "value": [ 17:44:06 DEBUG 1713545046, 17:44:06 DEBUG "0" 17:44:06 DEBUG ] 17:44:06 DEBUG } 17:44:06 DEBUG ] 17:44:06 DEBUG } 17:44:06 DEBUG } 17:44:06 INFO [http_cmd]: http status code OK 17:44:06 DEBUG --- status code --- 17:44:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:06 DEBUG --- http response --- 17:44:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:44:06 DEBUG ----- output ----- 17:44:06 DEBUG { 17:44:06 DEBUG "status": "success", 17:44:06 DEBUG "data": { 17:44:06 DEBUG "resultType": "vector", 17:44:06 DEBUG "result": [ 17:44:06 DEBUG { 17:44:06 DEBUG "metric": { 17:44:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:44:06 DEBUG }, 17:44:06 DEBUG "value": [ 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG [...] 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:44:06 DEBUG }, 17:44:06 DEBUG "value": [ 17:44:06 DEBUG 1713545046, 17:44:06 DEBUG "1442332672" 17:44:06 DEBUG ] 17:44:06 DEBUG } 17:44:06 DEBUG ] 17:44:06 DEBUG } 17:44:06 DEBUG } 17:44:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545046" 17:44:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545046" 17:44:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545046" 17:44:06 INFO [http_cmd]: http status code OK 17:44:06 DEBUG --- status code --- 17:44:06 INFO [http_cmd]: http status code OK 17:44:06 DEBUG --- status code --- 17:44:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:06 DEBUG --- http response --- 17:44:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:44:06 DEBUG ----- output ----- 17:44:06 DEBUG { 17:44:06 DEBUG "status": "success", 17:44:06 DEBUG "data": { 17:44:06 DEBUG "resultType": "vector", 17:44:06 DEBUG "result": [ 17:44:06 DEBUG { 17:44:06 DEBUG "metric": { 17:44:06 DEBUG "pod": "ds-cts-0" 17:44:06 DEBUG }, 17:44:06 DEBUG "value": [ 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG [...] 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG "pod": "ds-cts-2" 17:44:06 DEBUG }, 17:44:06 DEBUG "value": [ 17:44:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:06 DEBUG 1713545046, 17:44:06 DEBUG --- http response --- 17:44:06 DEBUG "0.28400000000000036" 17:44:06 DEBUG ] 17:44:06 DEBUG } 17:44:06 DEBUG ] 17:44:06 DEBUG } 17:44:06 DEBUG } 17:44:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:44:06 DEBUG ----- output ----- 17:44:06 DEBUG { 17:44:06 DEBUG "status": "success", 17:44:06 DEBUG "data": { 17:44:06 DEBUG "resultType": "vector", 17:44:06 DEBUG "result": [ 17:44:06 DEBUG { 17:44:06 DEBUG "metric": { 17:44:06 DEBUG "pod": "ds-idrepo-0" 17:44:06 DEBUG }, 17:44:06 DEBUG "value": [ 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG [...] 17:44:06 INFO [http_cmd]: http status code OK 17:44:06 DEBUG 17:44:06 DEBUG --- status code --- 17:44:06 DEBUG 17:44:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:06 DEBUG 17:44:06 DEBUG --- http response --- 17:44:06 DEBUG "pod": "ds-idrepo-2" 17:44:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:44:06 DEBUG }, 17:44:06 DEBUG ----- output ----- 17:44:06 DEBUG "value": [ 17:44:06 DEBUG { 17:44:06 DEBUG 1713545046, 17:44:06 DEBUG "status": "success", 17:44:06 DEBUG "0" 17:44:06 DEBUG "data": { 17:44:06 DEBUG ] 17:44:06 DEBUG "resultType": "vector", 17:44:06 DEBUG } 17:44:06 DEBUG "result": [ 17:44:06 DEBUG ] 17:44:06 DEBUG { 17:44:06 DEBUG } 17:44:06 DEBUG "metric": { 17:44:06 DEBUG } 17:44:06 DEBUG "pod": "ds-cts-0" 17:44:06 DEBUG }, 17:44:06 DEBUG "value": [ 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG [...] 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG "pod": "ds-cts-2" 17:44:06 DEBUG }, 17:44:06 DEBUG "value": [ 17:44:06 DEBUG 1713545046, 17:44:06 DEBUG "29.8" 17:44:06 DEBUG ] 17:44:06 DEBUG } 17:44:06 DEBUG ] 17:44:06 DEBUG } 17:44:06 DEBUG } 17:44:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545046" 17:44:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545046" 17:44:06 INFO [http_cmd]: http status code OK 17:44:06 DEBUG --- status code --- 17:44:06 INFO [http_cmd]: http status code OK 17:44:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:06 DEBUG --- status code --- 17:44:06 DEBUG --- http response --- 17:44:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:06 DEBUG --- http response --- 17:44:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:44:06 DEBUG ----- output ----- 17:44:06 DEBUG { 17:44:06 DEBUG "status": "success", 17:44:06 DEBUG "data": { 17:44:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:44:06 DEBUG "resultType": "vector", 17:44:06 DEBUG ----- output ----- 17:44:06 DEBUG "result": [ 17:44:06 DEBUG { 17:44:06 DEBUG { 17:44:06 DEBUG "status": "success", 17:44:06 DEBUG "metric": { 17:44:06 DEBUG "data": { 17:44:06 DEBUG "pod": "ds-idrepo-1" 17:44:06 DEBUG "resultType": "vector", 17:44:06 DEBUG }, 17:44:06 DEBUG "value": [ 17:44:06 DEBUG "result": [ 17:44:06 DEBUG 17:44:06 DEBUG { 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG [...] 17:44:06 DEBUG "metric": { 17:44:06 DEBUG "instance": "10.142.0.107:9100" 17:44:06 DEBUG 17:44:06 DEBUG }, 17:44:06 DEBUG 17:44:06 DEBUG "value": [ 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG }, 17:44:06 DEBUG [...] 17:44:06 DEBUG "value": [ 17:44:06 DEBUG 17:44:06 DEBUG 1713545046, 17:44:06 DEBUG 17:44:06 DEBUG "0" 17:44:06 DEBUG 17:44:06 DEBUG ] 17:44:06 DEBUG "instance": "10.142.0.43:9100" 17:44:06 DEBUG } 17:44:06 DEBUG }, 17:44:06 DEBUG ] 17:44:06 DEBUG "value": [ 17:44:06 DEBUG } 17:44:06 DEBUG 1713545046, 17:44:06 DEBUG } 17:44:06 DEBUG "75467827.5" 17:44:06 DEBUG ] 17:44:06 DEBUG } 17:44:06 DEBUG ] 17:44:06 DEBUG } 17:44:06 DEBUG } 17:44:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545046" 17:44:06 INFO [http_cmd]: http status code OK 17:44:06 DEBUG --- status code --- 17:44:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:06 DEBUG --- http response --- 17:44:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:44:06 DEBUG ----- output ----- 17:44:06 DEBUG { 17:44:06 DEBUG "status": "success", 17:44:06 DEBUG "data": { 17:44:06 DEBUG "resultType": "vector", 17:44:06 DEBUG "result": [ 17:44:06 DEBUG { 17:44:06 DEBUG "metric": { 17:44:06 DEBUG "instance": "10.142.0.107:9100" 17:44:06 DEBUG }, 17:44:06 DEBUG "value": [ 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG [...] 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG 17:44:06 DEBUG "instance": "10.142.0.43:9100" 17:44:06 DEBUG }, 17:44:06 DEBUG "value": [ 17:44:06 DEBUG 1713545046, 17:44:06 DEBUG "75460743.96666667" 17:44:06 DEBUG ] 17:44:06 DEBUG } 17:44:06 DEBUG ] 17:44:06 DEBUG } 17:44:06 DEBUG } 17:44:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545047" 17:44:07 INFO [http_cmd]: http status code OK 17:44:07 DEBUG --- status code --- 17:44:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:07 DEBUG --- http response --- 17:44:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:44:07 DEBUG ----- output ----- 17:44:07 DEBUG { 17:44:07 DEBUG "status": "success", 17:44:07 DEBUG "data": { 17:44:07 DEBUG "resultType": "vector", 17:44:07 DEBUG "result": [ 17:44:07 DEBUG { 17:44:07 DEBUG "metric": { 17:44:07 DEBUG "pod": "ds-idrepo-1" 17:44:07 DEBUG }, 17:44:07 DEBUG "value": [ 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG [...] 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:44:07 DEBUG }, 17:44:07 DEBUG "value": [ 17:44:07 DEBUG 1713545047, 17:44:07 DEBUG "0" 17:44:07 DEBUG ] 17:44:07 DEBUG } 17:44:07 DEBUG ] 17:44:07 DEBUG } 17:44:07 DEBUG } 17:44:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545047" 17:44:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545047" 17:44:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545047" 17:44:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545047" 17:44:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545047" 17:44:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545047" 17:44:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545047" 17:44:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545047" 17:44:07 INFO [http_cmd]: http status code OK 17:44:07 INFO [http_cmd]: http status code OK 17:44:07 DEBUG --- status code --- 17:44:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:07 INFO [http_cmd]: http status code OK 17:44:07 INFO [http_cmd]: http status code OK 17:44:07 INFO [http_cmd]: http status code OK 17:44:07 DEBUG --- status code --- 17:44:07 DEBUG --- http response --- 17:44:07 INFO [http_cmd]: http status code OK 17:44:07 DEBUG --- status code --- 17:44:07 DEBUG --- status code --- 17:44:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:44:07 DEBUG --- status code --- 17:44:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:07 DEBUG --- http response --- 17:44:07 DEBUG --- status code --- 17:44:07 DEBUG --- http response --- 17:44:07 DEBUG ----- output ----- 17:44:07 DEBUG { 17:44:07 INFO [http_cmd]: http status code OK 17:44:07 DEBUG --- http response --- 17:44:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:44:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:44:07 DEBUG "status": "success", 17:44:07 INFO [http_cmd]: http status code OK 17:44:07 DEBUG --- status code --- 17:44:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:44:07 DEBUG --- http response --- 17:44:07 DEBUG --- http response --- 17:44:07 DEBUG ----- output ----- 17:44:07 DEBUG { 17:44:07 DEBUG "data": { 17:44:07 DEBUG --- status code --- 17:44:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:07 DEBUG ----- output ----- 17:44:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:44:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:44:07 DEBUG ----- output ----- 17:44:07 DEBUG "status": "success", 17:44:07 DEBUG "resultType": "vector", 17:44:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:44:07 DEBUG --- http response --- 17:44:07 DEBUG { 17:44:07 DEBUG ----- output ----- 17:44:07 DEBUG ----- output ----- 17:44:07 DEBUG { 17:44:07 DEBUG "data": { 17:44:07 DEBUG "resultType": "vector", 17:44:07 DEBUG --- http response --- 17:44:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:44:07 DEBUG "status": "success", 17:44:07 DEBUG { 17:44:07 DEBUG { 17:44:07 DEBUG "status": "success", 17:44:07 DEBUG "result": [ 17:44:07 DEBUG "result": [ 17:44:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:44:07 DEBUG ----- output ----- 17:44:07 DEBUG "data": { 17:44:07 DEBUG "status": "success", 17:44:07 DEBUG "status": "success", 17:44:07 DEBUG "data": { 17:44:07 DEBUG { 17:44:07 DEBUG { 17:44:07 DEBUG ----- output ----- 17:44:07 DEBUG { 17:44:07 DEBUG "status": "success", 17:44:07 DEBUG "data": { 17:44:07 DEBUG "data": { 17:44:07 DEBUG "resultType": "vector", 17:44:07 DEBUG "metric": { 17:44:07 DEBUG "metric": { 17:44:07 DEBUG "data": { 17:44:07 DEBUG { 17:44:07 DEBUG "resultType": "vector", 17:44:07 DEBUG "resultType": "vector", 17:44:07 DEBUG "resultType": "vector", 17:44:07 DEBUG "result": [ 17:44:07 DEBUG { 17:44:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:44:07 DEBUG "resultType": "vector", 17:44:07 DEBUG "status": "success", 17:44:07 DEBUG "result": [ 17:44:07 DEBUG "result": [ 17:44:07 DEBUG "result": [ 17:44:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:44:07 DEBUG "metric": { 17:44:07 DEBUG }, 17:44:07 DEBUG "result": [ 17:44:07 DEBUG { 17:44:07 DEBUG { 17:44:07 DEBUG { 17:44:07 DEBUG "metric": { 17:44:07 DEBUG { 17:44:07 DEBUG "metric": { 17:44:07 DEBUG "value": [ 17:44:07 DEBUG "metric": { 17:44:07 DEBUG "metric": { 17:44:07 DEBUG "instance": "10.142.0.107:9100" 17:44:07 DEBUG "data": { 17:44:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:44:07 DEBUG 17:44:07 DEBUG "pod": "ds-idrepo-0" 17:44:07 DEBUG }, 17:44:07 DEBUG "value": [ 17:44:07 DEBUG }, 17:44:07 DEBUG "pod": "ds-cts-0" 17:44:07 DEBUG "resultType": "vector", 17:44:07 DEBUG }, 17:44:07 DEBUG 17:44:07 DEBUG }, 17:44:07 DEBUG 17:44:07 DEBUG "value": [ 17:44:07 DEBUG }, 17:44:07 DEBUG "result": [ 17:44:07 DEBUG "value": [ 17:44:07 DEBUG 17:44:07 DEBUG "value": [ 17:44:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG [...] 17:44:07 DEBUG { 17:44:07 DEBUG "metric": { 17:44:07 DEBUG [...] 17:44:07 DEBUG 17:44:07 DEBUG }, 17:44:07 DEBUG "value": [ 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:44:07 DEBUG }, 17:44:07 DEBUG "value": [ 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG [...] 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG [...] 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG "value": [ 17:44:07 DEBUG 17:44:07 DEBUG [...] 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG [...] 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:44:07 DEBUG }, 17:44:07 DEBUG 17:44:07 DEBUG [...] 17:44:07 DEBUG }, 17:44:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:44:07 DEBUG "pod": "ds-idrepo-2" 17:44:07 DEBUG 17:44:07 DEBUG "value": [ 17:44:07 DEBUG 17:44:07 DEBUG "instance": "10.142.0.43:9100" 17:44:07 DEBUG 17:44:07 DEBUG 17:44:07 DEBUG }, 17:44:07 DEBUG }, 17:44:07 DEBUG "value": [ 17:44:07 DEBUG 1713545047, 17:44:07 DEBUG 17:44:07 DEBUG }, 17:44:07 DEBUG 17:44:07 DEBUG "value": [ 17:44:07 DEBUG 17:44:07 DEBUG "value": [ 17:44:07 DEBUG 1713545047, 17:44:07 DEBUG "2503.366666666667" 17:44:07 DEBUG [...] 17:44:07 DEBUG "value": [ 17:44:07 DEBUG "pod": "ds-idrepo-2" 17:44:07 DEBUG 1713545047, 17:44:07 DEBUG "8.041462025941218" 17:44:07 DEBUG 1713545047, 17:44:07 DEBUG "pod": "ds-cts-2" 17:44:07 DEBUG ] 17:44:07 DEBUG 17:44:07 DEBUG 1713545047, 17:44:07 DEBUG }, 17:44:07 DEBUG ] 17:44:07 DEBUG "0.023" 17:44:07 DEBUG "0" 17:44:07 DEBUG }, 17:44:07 DEBUG "value": [ 17:44:07 DEBUG 17:44:07 DEBUG "0.8860666666666428" 17:44:07 DEBUG "value": [ 17:44:07 DEBUG } 17:44:07 DEBUG } 17:44:07 DEBUG ] 17:44:07 DEBUG 1713545047, 17:44:07 DEBUG 17:44:07 DEBUG ] 17:44:07 DEBUG 1713545047, 17:44:07 DEBUG ] 17:44:07 DEBUG ] 17:44:07 DEBUG } 17:44:07 DEBUG } 17:44:07 DEBUG "241" 17:44:07 DEBUG } 17:44:07 DEBUG } 17:44:07 DEBUG ] 17:44:07 DEBUG } 17:44:07 DEBUG "7.084053608385048" 17:44:07 DEBUG ] 17:44:07 DEBUG ] 17:44:07 DEBUG ] 17:44:07 DEBUG } 17:44:07 DEBUG ] 17:44:07 DEBUG ] 17:44:07 DEBUG } 17:44:07 DEBUG } 17:44:07 DEBUG } 17:44:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:44:07 DEBUG } 17:44:07 DEBUG } 17:44:07 DEBUG } 17:44:07 DEBUG } 17:44:07 DEBUG ] 17:44:07 DEBUG } 17:44:07 DEBUG }, 17:44:07 DEBUG } 17:44:07 DEBUG ] 17:44:07 DEBUG } 17:44:07 DEBUG "value": [ 17:44:07 DEBUG } 17:44:07 DEBUG } 17:44:07 DEBUG 1713545047, 17:44:07 DEBUG } 17:44:07 DEBUG "2504" 17:44:07 DEBUG ] 17:44:07 DEBUG } 17:44:07 DEBUG ] 17:44:07 DEBUG } 17:44:07 DEBUG } 17:44:07 INFO 17:44:07 INFO [loop_until]: kubectl --namespace=xlou top pods 17:44:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:44:08 INFO [loop_until]: OK (rc = 0) 17:44:08 DEBUG --- stdout --- 17:44:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 16931m 9002Mi am-6fd554f8f7-btd4s 16198m 9041Mi am-6fd554f8f7-wxcsj 18469m 9028Mi ds-cts-0 20665m 12799Mi ds-cts-1 21654m 12814Mi ds-cts-2 21976m 12815Mi ds-idrepo-0 6833m 3422Mi ds-idrepo-1 6862m 3394Mi ds-idrepo-2 7236m 3461Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8029m 1289Mi 17:44:08 DEBUG --- stderr --- 17:44:08 DEBUG 17:44:08 INFO 17:44:08 INFO [loop_until]: kubectl --namespace=xlou top node 17:44:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:44:08 INFO [loop_until]: OK (rc = 0) 17:44:08 DEBUG --- stdout --- 17:44:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6162Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7447m 24% 7004Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 45m 0% 2683Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17624m 59% 12298Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 20919m 70% 15837Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 135m 0% 5238Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17379m 58% 11940Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16926m 56% 3033Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18972m 63% 11258Mi 10% gke-xlou-cdm-ds-0f601751-j488 7701m 25% 6700Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 21282m 71% 17563Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7435m 24% 7237Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 20751m 69% 15253Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 7904m 49% 5024Mi 8% 17:44:08 DEBUG --- stderr --- 17:44:08 DEBUG 17:45:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545104" 17:45:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545104" 17:45:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545104" 17:45:04 INFO [http_cmd]: http status code OK 17:45:04 DEBUG --- status code --- 17:45:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:04 DEBUG --- http response --- 17:45:04 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:45:04 ERROR [http_cmd]: ERROR 17:45:04 INFO [http_cmd]: http status code OK 17:45:04 ERROR --- status code --- 17:45:04 DEBUG --- status code --- 17:45:04 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:45:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:04 ERROR --- http response --- 17:45:04 DEBUG --- http response --- 17:45:04 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:45:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:45:04 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:45:04 DEBUG ----- output ----- 17:45:04 DEBUG { 17:45:04 DEBUG "status": "success", 17:45:04 DEBUG "data": { 17:45:04 DEBUG "resultType": "vector", 17:45:04 DEBUG "result": [ 17:45:04 DEBUG { 17:45:04 DEBUG "metric": { 17:45:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:45:04 DEBUG }, 17:45:04 DEBUG "value": [ 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG [...] 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:45:04 DEBUG }, 17:45:04 DEBUG "value": [ 17:45:04 DEBUG 1713545104, 17:45:04 DEBUG "2366.0333333333333" 17:45:04 DEBUG ] 17:45:04 DEBUG } 17:45:04 DEBUG ] 17:45:04 DEBUG } 17:45:04 DEBUG } 17:45:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545104" 17:45:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545104" 17:45:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545104" 17:45:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545104" 17:45:04 INFO [http_cmd]: http status code OK 17:45:04 DEBUG --- status code --- 17:45:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:04 DEBUG --- http response --- 17:45:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:45:04 DEBUG ----- output ----- 17:45:04 DEBUG { 17:45:04 DEBUG "status": "success", 17:45:04 DEBUG "data": { 17:45:04 DEBUG "resultType": "vector", 17:45:04 DEBUG "result": [ 17:45:04 DEBUG { 17:45:04 DEBUG "metric": { 17:45:04 DEBUG "pod": "ds-idrepo-0" 17:45:04 DEBUG }, 17:45:04 DEBUG "value": [ 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG [...] 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG "pod": "ds-cts-2" 17:45:04 DEBUG }, 17:45:04 DEBUG "value": [ 17:45:04 DEBUG 1713545104, 17:45:04 DEBUG "0" 17:45:04 DEBUG ] 17:45:04 DEBUG } 17:45:04 DEBUG ] 17:45:04 DEBUG } 17:45:04 DEBUG } 17:45:04 INFO [http_cmd]: http status code OK 17:45:04 DEBUG --- status code --- 17:45:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:04 DEBUG --- http response --- 17:45:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:45:04 DEBUG ----- output ----- 17:45:04 DEBUG { 17:45:04 DEBUG "status": "success", 17:45:04 DEBUG "data": { 17:45:04 DEBUG "resultType": "vector", 17:45:04 DEBUG "result": [ 17:45:04 DEBUG { 17:45:04 DEBUG "metric": { 17:45:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:45:04 DEBUG }, 17:45:04 DEBUG "value": [ 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG [...] 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:45:04 DEBUG }, 17:45:04 DEBUG "value": [ 17:45:04 DEBUG 1713545104, 17:45:04 DEBUG "0" 17:45:04 DEBUG ] 17:45:04 DEBUG } 17:45:04 DEBUG ] 17:45:04 DEBUG } 17:45:04 DEBUG } 17:45:04 INFO [http_cmd]: http status code OK 17:45:04 DEBUG --- status code --- 17:45:04 INFO [http_cmd]: http status code OK 17:45:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:04 DEBUG --- status code --- 17:45:04 DEBUG --- http response --- 17:45:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:45:04 DEBUG ----- output ----- 17:45:04 DEBUG --- http response --- 17:45:04 DEBUG { 17:45:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:45:04 DEBUG "status": "success", 17:45:04 DEBUG ----- output ----- 17:45:04 DEBUG "data": { 17:45:04 DEBUG { 17:45:04 DEBUG "resultType": "vector", 17:45:04 DEBUG "status": "success", 17:45:04 DEBUG "result": [ 17:45:04 DEBUG "data": { 17:45:04 DEBUG { 17:45:04 DEBUG "resultType": "vector", 17:45:04 DEBUG "metric": { 17:45:04 DEBUG "result": [ 17:45:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:45:04 DEBUG { 17:45:04 DEBUG }, 17:45:04 DEBUG "metric": { 17:45:04 DEBUG "value": [ 17:45:04 DEBUG "pod": "ds-idrepo-0" 17:45:04 DEBUG 17:45:04 DEBUG }, 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG "value": [ 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG [...] 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG 17:45:04 DEBUG "pod": "ds-idrepo-2" 17:45:04 DEBUG [...] 17:45:04 DEBUG }, 17:45:04 DEBUG 17:45:04 DEBUG "value": [ 17:45:04 DEBUG 17:45:04 DEBUG 1713545104, 17:45:04 DEBUG 17:45:04 DEBUG "0" 17:45:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:45:04 DEBUG ] 17:45:04 DEBUG }, 17:45:04 DEBUG } 17:45:04 DEBUG "value": [ 17:45:04 DEBUG ] 17:45:04 DEBUG 1713545104, 17:45:04 DEBUG } 17:45:04 DEBUG "2498.8" 17:45:04 DEBUG } 17:45:04 DEBUG ] 17:45:04 DEBUG } 17:45:04 DEBUG ] 17:45:04 DEBUG } 17:45:04 DEBUG } 17:45:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545105" 17:45:05 INFO [http_cmd]: http status code OK 17:45:05 DEBUG --- status code --- 17:45:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:05 DEBUG --- http response --- 17:45:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:45:05 DEBUG ----- output ----- 17:45:05 DEBUG { 17:45:05 DEBUG "status": "success", 17:45:05 DEBUG "data": { 17:45:05 DEBUG "resultType": "vector", 17:45:05 DEBUG "result": [ 17:45:05 DEBUG { 17:45:05 DEBUG "metric": { 17:45:05 DEBUG "instance": "10.142.0.107:9100" 17:45:05 DEBUG }, 17:45:05 DEBUG "value": [ 17:45:05 DEBUG 17:45:05 DEBUG 17:45:05 DEBUG 17:45:05 DEBUG [...] 17:45:05 DEBUG 17:45:05 DEBUG 17:45:05 DEBUG 17:45:05 DEBUG "instance": "10.142.0.43:9100" 17:45:05 DEBUG }, 17:45:05 DEBUG "value": [ 17:45:05 DEBUG 1713545105, 17:45:05 DEBUG "0.6363333333333381" 17:45:05 DEBUG ] 17:45:05 DEBUG } 17:45:05 DEBUG ] 17:45:05 DEBUG } 17:45:05 DEBUG } 17:45:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545105" 17:45:05 INFO [http_cmd]: http status code OK 17:45:05 DEBUG --- status code --- 17:45:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:05 DEBUG --- http response --- 17:45:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:45:05 DEBUG ----- output ----- 17:45:05 DEBUG { 17:45:05 DEBUG "status": "success", 17:45:05 DEBUG "data": { 17:45:05 DEBUG[0m "resultType": "vector", 17:45:05 DEBUG "result": [ 17:45:05 DEBUG { 17:45:05 DEBUG "metric": { 17:45:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:45:05 DEBUG }, 17:45:05 DEBUG "value": [ 17:45:05 DEBUG 17:45:05 DEBUG 17:45:05 DEBUG 17:45:05 DEBUG [...] 17:45:05 DEBUG 17:45:05 DEBUG 17:45:05 DEBUG 17:45:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:45:05 DEBUG }, 17:45:05 DEBUG "value": [ 17:45:05 DEBUG 1713545105, 17:45:05 DEBUG "4196088" 17:45:05 DEBUG ] 17:45:05 DEBUG } 17:45:05 DEBUG ] 17:45:05 DEBUG } 17:45:05 DEBUG } 17:45:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545105" 17:45:05 INFO [http_cmd]: http status code OK 17:45:05 DEBUG --- status code --- 17:45:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:05 DEBUG --- http response --- 17:45:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:45:05 DEBUG ----- output ----- 17:45:05 DEBUG { 17:45:05 DEBUG "status": "success", 17:45:05 DEBUG "data": { 17:45:05 DEBUG "resultType": "vector", 17:45:05 DEBUG "result": [ 17:45:05 DEBUG { 17:45:05 DEBUG "metric": { 17:45:05 DEBUG "instance": "10.142.0.107:9100" 17:45:05 DEBUG }, 17:45:05 DEBUG "value": [ 17:45:05 DEBUG 17:45:05 DEBUG 17:45:05 DEBUG 17:45:05 DEBUG [...] 17:45:05 DEBUG 17:45:05 DEBUG 17:45:05 DEBUG 17:45:05 DEBUG "instance": "10.142.0.43:9100" 17:45:05 DEBUG }, 17:45:05 DEBUG "value": [ 17:45:05 DEBUG 1713545105, 17:45:05 DEBUG "1760810839040" 17:45:05 DEBUG ] 17:45:05 DEBUG } 17:45:05 DEBUG ] 17:45:05 DEBUG } 17:45:05 DEBUG } 17:45:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545106" 17:45:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545106" 17:45:06 INFO [http_cmd]: http status code OK 17:45:06 DEBUG --- status code --- 17:45:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:06 DEBUG --- http response --- 17:45:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:45:06 DEBUG ----- output ----- 17:45:06 DEBUG { 17:45:06 DEBUG "status": "success", 17:45:06 DEBUG "data": { 17:45:06 DEBUG "resultType": "vector", 17:45:06 DEBUG "result": [ 17:45:06 DEBUG { 17:45:06 DEBUG "metric": { 17:45:06 DEBUG "pod": "ds-idrepo-0" 17:45:06 DEBUG }, 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG [...] 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG "pod": "ds-idrepo-2" 17:45:06 DEBUG }, 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 1713545106, 17:45:06 DEBUG "0" 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG } 17:45:06 INFO [http_cmd]: http status code OK 17:45:06 DEBUG --- status code --- 17:45:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:06 DEBUG --- http response --- 17:45:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:45:06 DEBUG ----- output ----- 17:45:06 DEBUG { 17:45:06 DEBUG "status": "success", 17:45:06 DEBUG "data": { 17:45:06 DEBUG "resultType": "vector", 17:45:06 DEBUG "result": [ 17:45:06 DEBUG { 17:45:06 DEBUG "metric": { 17:45:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:45:06 DEBUG }, 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG [...] 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:45:06 DEBUG }, 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 1713545106, 17:45:06 DEBUG "1695100928" 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG } 17:45:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545106" 17:45:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545106" 17:45:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545106" 17:45:06 INFO [http_cmd]: http status code OK 17:45:06 DEBUG --- status code --- 17:45:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:06 DEBUG --- http response --- 17:45:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:45:06 DEBUG ----- output ----- 17:45:06 DEBUG { 17:45:06 DEBUG "status": "success", 17:45:06 DEBUG "data": { 17:45:06 DEBUG "resultType": "vector", 17:45:06 DEBUG "result": [ 17:45:06 DEBUG { 17:45:06 DEBUG "metric": { 17:45:06 DEBUG "pod": "ds-cts-0" 17:45:06 DEBUG }, 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG [...] 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG "pod": "ds-cts-2" 17:45:06 DEBUG }, 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 1713545106, 17:45:06 DEBUG "0.33609999999999995" 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG } 17:45:06 INFO [http_cmd]: http status code OK 17:45:06 INFO [http_cmd]: http status code OK 17:45:06 DEBUG --- status code --- 17:45:06 DEBUG --- status code --- 17:45:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:06 DEBUG --- http response --- 17:45:06 DEBUG --- http response --- 17:45:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:45:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:45:06 DEBUG ----- output ----- 17:45:06 DEBUG ----- output ----- 17:45:06 DEBUG { 17:45:06 DEBUG { 17:45:06 DEBUG "status": "success", 17:45:06 DEBUG "status": "success", 17:45:06 DEBUG "data": { 17:45:06 DEBUG "data": { 17:45:06 DEBUG "resultType": "vector", 17:45:06 DEBUG "resultType": "vector", 17:45:06 DEBUG "result": [ 17:45:06 DEBUG "result": [ 17:45:06 DEBUG { 17:45:06 DEBUG { 17:45:06 DEBUG "metric": { 17:45:06 DEBUG "metric": { 17:45:06 DEBUG "pod": "ds-idrepo-0" 17:45:06 DEBUG "pod": "ds-cts-0" 17:45:06 DEBUG }, 17:45:06 DEBUG }, 17:45:06 DEBUG "value": [ 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG [...] 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG "pod": "ds-idrepo-2" 17:45:06 DEBUG }, 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 1713545106, 17:45:06 DEBUG "0" 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG } 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG [...] 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG "pod": "ds-cts-2" 17:45:06 DEBUG }, 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 1713545106, 17:45:06 DEBUG "30.5" 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG } 17:45:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545106" 17:45:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545106" 17:45:06 INFO [http_cmd]: http status code OK 17:45:06 DEBUG --- status code --- 17:45:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:06 DEBUG --- http response --- 17:45:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:45:06 INFO [http_cmd]: http status code OK 17:45:06 DEBUG ----- output ----- 17:45:06 DEBUG --- status code --- 17:45:06 DEBUG { 17:45:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:06 DEBUG "status": "success", 17:45:06 DEBUG --- http response --- 17:45:06 DEBUG "data": { 17:45:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:45:06 DEBUG "resultType": "vector", 17:45:06 DEBUG ----- output ----- 17:45:06 DEBUG "result": [ 17:45:06 DEBUG { 17:45:06 DEBUG { 17:45:06 DEBUG "status": "success", 17:45:06 DEBUG "metric": { 17:45:06 DEBUG "data": { 17:45:06 DEBUG "pod": "ds-idrepo-0" 17:45:06 DEBUG "resultType": "vector", 17:45:06 DEBUG }, 17:45:06 DEBUG "result": [ 17:45:06 DEBUG "value": [ 17:45:06 DEBUG { 17:45:06 DEBUG 17:45:06 DEBUG "metric": { 17:45:06 DEBUG 17:45:06 DEBUG "instance": "10.142.0.107:9100" 17:45:06 DEBUG 17:45:06 DEBUG }, 17:45:06 DEBUG [...] 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 17:45:06 DEBUG [...] 17:45:06 DEBUG }, 17:45:06 DEBUG 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 17:45:06 DEBUG 1713545106, 17:45:06 DEBUG 17:45:06 DEBUG "0" 17:45:06 DEBUG "instance": "10.142.0.43:9100" 17:45:06 DEBUG ] 17:45:06 DEBUG }, 17:45:06 DEBUG } 17:45:06 DEBUG "value": [ 17:45:06 DEBUG ] 17:45:06 DEBUG 1713545106, 17:45:06 DEBUG } 17:45:06 DEBUG "75280327.20000002" 17:45:06 DEBUG } 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG } 17:45:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545106" 17:45:06 INFO [http_cmd]: http status code OK 17:45:06 DEBUG --- status code --- 17:45:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:06 DEBUG --- http response --- 17:45:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:45:06 DEBUG ----- output ----- 17:45:06 DEBUG { 17:45:06 DEBUG "status": "success", 17:45:06 DEBUG "data": { 17:45:06 DEBUG "resultType": "vector", 17:45:06 DEBUG "result": [ 17:45:06 DEBUG { 17:45:06 DEBUG "metric": { 17:45:06 DEBUG "instance": "10.142.0.107:9100" 17:45:06 DEBUG }, 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG [...] 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG 17:45:06 DEBUG "instance": "10.142.0.43:9100" 17:45:06 DEBUG }, 17:45:06 DEBUG "value": [ 17:45:06 DEBUG 1713545106, 17:45:06 DEBUG "75275377.26666668" 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG ] 17:45:06 DEBUG } 17:45:06 DEBUG } 17:45:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545107" 17:45:07 INFO [http_cmd]: http status code OK 17:45:07 DEBUG --- status code --- 17:45:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:07 DEBUG --- http response --- 17:45:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:45:07 DEBUG ----- output ----- 17:45:07 DEBUG { 17:45:07 DEBUG "status": "success", 17:45:07 DEBUG "data": { 17:45:07 DEBUG "resultType": "vector", 17:45:07 DEBUG "result": [ 17:45:07 DEBUG { 17:45:07 DEBUG "metric": { 17:45:07 DEBUG "pod": "ds-idrepo-0" 17:45:07 DEBUG }, 17:45:07 DEBUG "value": [ 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG [...] 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 17:45:07 DEBUG }, 17:45:07 DEBUG "value": [ 17:45:07 DEBUG 1713545107, 17:45:07 DEBUG "0" 17:45:07 DEBUG ] 17:45:07 DEBUG } 17:45:07 DEBUG ] 17:45:07 DEBUG } 17:45:07 DEBUG } 17:45:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545107" 17:45:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545107" 17:45:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545107" 17:45:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545107" 17:45:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545107" 17:45:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545107" 17:45:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545107" 17:45:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545107" 17:45:07 INFO [http_cmd]: http status code OK 17:45:07 DEBUG --- status code --- 17:45:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:07 DEBUG --- http response --- 17:45:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:45:07 DEBUG ----- output ----- 17:45:07 DEBUG { 17:45:07 DEBUG "status": "success", 17:45:07 DEBUG "data": { 17:45:07 DEBUG "resultType": "vector", 17:45:07 DEBUG "result": [ 17:45:07 INFO [http_cmd]: http status code OK 17:45:07 INFO [http_cmd]: http status code OK 17:45:07 INFO [http_cmd]: http status code OK 17:45:07 INFO [http_cmd]: http status code OK 17:45:07 INFO [http_cmd]: http status code OK 17:45:07 DEBUG --- status code --- 17:45:07 DEBUG --- status code --- 17:45:07 DEBUG --- status code --- 17:45:07 DEBUG --- status code --- 17:45:07 DEBUG --- status code --- 17:45:07 DEBUG { 17:45:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:07 INFO [http_cmd]: http status code OK 17:45:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:07 INFO [http_cmd]: http status code OK 17:45:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:07 DEBUG "metric": { 17:45:07 DEBUG --- http response --- 17:45:07 DEBUG --- status code --- 17:45:07 DEBUG --- http response --- 17:45:07 DEBUG --- http response --- 17:45:07 DEBUG --- status code --- 17:45:07 DEBUG --- http response --- 17:45:07 DEBUG --- http response --- 17:45:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:45:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:45:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:45:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:45:07 DEBUG --- http response --- 17:45:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:45:07 DEBUG }, 17:45:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:45:07 DEBUG ----- output ----- 17:45:07 DEBUG --- http response --- 17:45:07 DEBUG ----- output ----- 17:45:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:45:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:45:07 DEBUG ----- output ----- 17:45:07 DEBUG "value": [ 17:45:07 DEBUG ----- output ----- 17:45:07 DEBUG { 17:45:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:45:07 DEBUG { 17:45:07 DEBUG ----- output ----- 17:45:07 DEBUG ----- output ----- 17:45:07 DEBUG { 17:45:07 DEBUG "status": "success", 17:45:07 DEBUG { 17:45:07 DEBUG "status": "success", 17:45:07 DEBUG ----- output ----- 17:45:07 DEBUG "status": "success", 17:45:07 DEBUG { 17:45:07 DEBUG { 17:45:07 DEBUG 17:45:07 DEBUG "data": { 17:45:07 DEBUG "status": "success", 17:45:07 DEBUG "data": { 17:45:07 DEBUG { 17:45:07 DEBUG "data": { 17:45:07 DEBUG "status": "success", 17:45:07 DEBUG "status": "success", 17:45:07 DEBUG 17:45:07 DEBUG "resultType": "vector", 17:45:07 DEBUG "data": { 17:45:07 DEBUG "resultType": "vector", 17:45:07 DEBUG "status": "success", 17:45:07 DEBUG "resultType": "vector", 17:45:07 DEBUG "data": { 17:45:07 DEBUG "data": { 17:45:07 DEBUG "resultType": "vector", 17:45:07 DEBUG "result": [ 17:45:07 DEBUG "resultType": "vector", 17:45:07 DEBUG "result": [ 17:45:07 DEBUG { 17:45:07 DEBUG "result": [ 17:45:07 DEBUG "resultType": "vector", 17:45:07 DEBUG "result": [ 17:45:07 DEBUG 17:45:07 DEBUG { 17:45:07 DEBUG "result": [ 17:45:07 DEBUG "metric": { 17:45:07 DEBUG "data": { 17:45:07 DEBUG { 17:45:07 DEBUG "result": [ 17:45:07 DEBUG { 17:45:07 DEBUG [...] 17:45:07 DEBUG "metric": { 17:45:07 DEBUG { 17:45:07 DEBUG "pod": "ds-idrepo-0" 17:45:07 DEBUG "resultType": "vector", 17:45:07 DEBUG "metric": { 17:45:07 DEBUG { 17:45:07 DEBUG "metric": { 17:45:07 DEBUG 17:45:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:45:07 DEBUG "metric": { 17:45:07 DEBUG }, 17:45:07 DEBUG "result": [ 17:45:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:45:07 DEBUG "metric": { 17:45:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:45:07 DEBUG 17:45:07 DEBUG }, 17:45:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:45:07 DEBUG "value": [ 17:45:07 DEBUG { 17:45:07 DEBUG }, 17:45:07 DEBUG "instance": "10.142.0.107:9100" 17:45:07 DEBUG }, 17:45:07 DEBUG 17:45:07 DEBUG "value": [ 17:45:07 DEBUG }, 17:45:07 DEBUG 17:45:07 DEBUG "metric": { 17:45:07 DEBUG "value": [ 17:45:07 DEBUG }, 17:45:07 DEBUG "value": [ 17:45:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:45:07 DEBUG 17:45:07 DEBUG "value": [ 17:45:07 DEBUG 17:45:07 DEBUG "pod": "ds-cts-0" 17:45:07 DEBUG 17:45:07 DEBUG "value": [ 17:45:07 DEBUG 17:45:07 DEBUG }, 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG }, 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG "value": [ 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG [...] 17:45:07 DEBUG "value": [ 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 1713545107, 17:45:07 DEBUG [...] 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG [...] 17:45:07 DEBUG 17:45:07 DEBUG [...] 17:45:07 DEBUG "0.023" 17:45:07 DEBUG 17:45:07 DEBUG [...] 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG [...] 17:45:07 DEBUG 17:45:07 DEBUG ] 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG } 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG "pod": "ds-idrepo-2" 17:45:07 DEBUG [...] 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG 17:45:07 DEBUG ] 17:45:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:45:07 DEBUG 17:45:07 DEBUG }, 17:45:07 DEBUG 17:45:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:45:07 DEBUG 17:45:07 DEBUG "pod": "ds-idrepo-2" 17:45:07 DEBUG } 17:45:07 DEBUG }, 17:45:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:45:07 DEBUG "value": [ 17:45:07 DEBUG 17:45:07 DEBUG }, 17:45:07 DEBUG "instance": "10.142.0.43:9100" 17:45:07 DEBUG }, 17:45:07 DEBUG } 17:45:07 DEBUG "value": [ 17:45:07 DEBUG }, 17:45:07 DEBUG 1713545107, 17:45:07 DEBUG 17:45:07 DEBUG "value": [ 17:45:07 DEBUG }, 17:45:07 DEBUG "value": [ 17:45:07 DEBUG 1713545107, 17:45:07 DEBUG "value": [ 17:45:07 DEBUG "0" 17:45:07 DEBUG "pod": "ds-cts-2" 17:45:07 DEBUG 1713545107, 17:45:07 DEBUG "value": [ 17:45:07 DEBUG 1713545107, 17:45:07 DEBUG "2500.633333333333" 17:45:07 DEBUG 1713545107, 17:45:07 DEBUG ] 17:45:07 DEBUG }, 17:45:07 DEBUG "2510.3" 17:45:07 DEBUG 1713545107, 17:45:07 DEBUG "0" 17:45:07 DEBUG ] 17:45:07 DEBUG "8.034772988663546" 17:45:07 DEBUG } 17:45:07 DEBUG "value": [ 17:45:07 DEBUG ] 17:45:07 DEBUG "0.9356333333332287" 17:45:07 DEBUG ] 17:45:07 DEBUG } 17:45:07 DEBUG ] 17:45:07 DEBUG ] 17:45:07 DEBUG 1713545107, 17:45:07 DEBUG } 17:45:07 DEBUG ] 17:45:07 DEBUG } 17:45:07 DEBUG ] 17:45:07 DEBUG } 17:45:07 DEBUG } 17:45:07 DEBUG "246.2" 17:45:07 DEBUG ] 17:45:07 DEBUG } 17:45:07 DEBUG ] 17:45:07 DEBUG } 17:45:07 DEBUG ] 17:45:07 DEBUG } 17:45:07 DEBUG ] 17:45:07 DEBUG } 17:45:07 DEBUG ] 17:45:07 DEBUG } 17:45:07 DEBUG } 17:45:07 DEBUG } 17:45:07 DEBUG } 17:45:07 DEBUG } 17:45:07 DEBUG } 17:45:07 DEBUG } 17:45:07 DEBUG } 17:45:07 DEBUG ] 17:45:07 DEBUG } 17:45:07 DEBUG } 17:45:07 DEBUG } 17:45:08 INFO 17:45:08 INFO [loop_until]: kubectl --namespace=xlou top pods 17:45:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:45:08 INFO [loop_until]: OK (rc = 0) 17:45:08 DEBUG --- stdout --- 17:45:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17826m 9019Mi am-6fd554f8f7-btd4s 17906m 9068Mi am-6fd554f8f7-wxcsj 18813m 9052Mi ds-cts-0 20683m 12954Mi ds-cts-1 20766m 12820Mi ds-cts-2 21703m 13181Mi ds-idrepo-0 7018m 3423Mi ds-idrepo-1 6873m 3394Mi ds-idrepo-2 7108m 3461Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7943m 1672Mi 17:45:08 DEBUG --- stderr --- 17:45:08 DEBUG 17:45:08 INFO 17:45:08 INFO [loop_until]: kubectl --namespace=xlou top node 17:45:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:45:08 INFO [loop_until]: OK (rc = 0) 17:45:08 DEBUG --- stdout --- 17:45:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6172Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7417m 24% 7001Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 54m 0% 2684Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17753m 59% 12324Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 20139m 67% 16058Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5227Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17667m 59% 11976Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 17076m 57% 3098Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19671m 65% 11283Mi 10% gke-xlou-cdm-ds-0f601751-j488 7701m 25% 6702Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 21051m 70% 17531Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 7436m 24% 7237Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21432m 71% 15633Mi 14% gke-xlou-cdm-frontend-a8771548-i82n 8188m 51% 5396Mi 9% 17:45:08 DEBUG --- stderr --- 17:45:08 DEBUG 17:46:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545164" 17:46:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545164" 17:46:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545164" 17:46:04 ERROR [http_cmd]: ERROR 17:46:04 INFO [http_cmd]: http status code OK 17:46:04 INFO [http_cmd]: http status code OK 17:46:04 ERROR --- status code --- 17:46:04 DEBUG --- status code --- 17:46:04 DEBUG --- status code --- 17:46:04 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:46:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:04 ERROR --- http response --- 17:46:04 DEBUG --- http response --- 17:46:04 DEBUG --- http response --- 17:46:04 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:46:04 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:46:04 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:46:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:46:04 DEBUG ----- output ----- 17:46:04 DEBUG { 17:46:04 DEBUG "status": "success", 17:46:04 DEBUG "data": { 17:46:04 DEBUG "resultType": "vector", 17:46:04 DEBUG "result": [ 17:46:04 DEBUG { 17:46:04 DEBUG "metric": { 17:46:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:46:04 DEBUG }, 17:46:04 DEBUG "value": [ 17:46:04 DEBUG 17:46:04 DEBUG 17:46:04 DEBUG 17:46:04 DEBUG [...] 17:46:04 DEBUG 17:46:04 DEBUG 17:46:04 DEBUG 17:46:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:46:04 DEBUG }, 17:46:04 DEBUG "value": [ 17:46:04 DEBUG 1713545164, 17:46:04 DEBUG "2302.5" 17:46:04 DEBUG ] 17:46:04 DEBUG } 17:46:04 DEBUG ] 17:46:04 DEBUG } 17:46:04 DEBUG } 17:46:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545164" 17:46:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545164" 17:46:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545164" 17:46:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545164" 17:46:04 INFO [http_cmd]: http status code OK 17:46:04 DEBUG --- status code --- 17:46:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:04 DEBUG --- http response --- 17:46:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:46:04 DEBUG ----- output ----- 17:46:04 INFO [http_cmd]: http status code OK 17:46:04 DEBUG { 17:46:04 DEBUG --- status code --- 17:46:04 DEBUG "status": "success", 17:46:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:04 DEBUG "data": { 17:46:04 DEBUG --- http response --- 17:46:04 DEBUG "resultType": "vector", 17:46:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:46:04 DEBUG "result": [ 17:46:04 DEBUG { 17:46:04 DEBUG "metric": { 17:46:04 DEBUG "pod": "ds-idrepo-0" 17:46:04 DEBUG }, 17:46:04 DEBUG ----- output ----- 17:46:04 DEBUG "value": [ 17:46:04 DEBUG { 17:46:04 DEBUG 17:46:04 DEBUG "status": "success", 17:46:04 DEBUG 17:46:04 DEBUG "data": { 17:46:04 DEBUG 17:46:04 DEBUG "resultType": "vector", 17:46:04 DEBUG [...] 17:46:04 DEBUG "result": [ 17:46:04 DEBUG 17:46:04 DEBUG { 17:46:04 DEBUG 17:46:04 DEBUG "metric": { 17:46:04 DEBUG 17:46:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:46:04 DEBUG "pod": "ds-cts-2" 17:46:04 DEBUG }, 17:46:04 DEBUG }, 17:46:04 DEBUG "value": [ 17:46:04 DEBUG 17:46:04 DEBUG "value": [ 17:46:04 DEBUG 17:46:04 DEBUG 1713545164, 17:46:04 DEBUG 17:46:04 DEBUG "0" 17:46:04 DEBUG [...] 17:46:04 DEBUG ] 17:46:04 DEBUG 17:46:04 DEBUG } 17:46:04 DEBUG 17:46:04 DEBUG ] 17:46:04 DEBUG 17:46:04 DEBUG } 17:46:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:46:04 DEBUG } 17:46:04 DEBUG }, 17:46:04 DEBUG "value": [ 17:46:04 DEBUG 1713545164, 17:46:04 DEBUG "0" 17:46:04 DEBUG ] 17:46:04 DEBUG } 17:46:04 DEBUG ] 17:46:04 DEBUG } 17:46:04 DEBUG } 17:46:04 INFO [http_cmd]: http status code OK 17:46:04 INFO [http_cmd]: http status code OK 17:46:04 DEBUG --- status code --- 17:46:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:04 DEBUG --- http response --- 17:46:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:46:04 DEBUG ----- output ----- 17:46:04 DEBUG { 17:46:04 DEBUG --- status code --- 17:46:04 DEBUG "status": "success", 17:46:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:04 DEBUG "data": { 17:46:04 DEBUG --- http response --- 17:46:04 DEBUG "resultType": "vector", 17:46:04 DEBUG "result": [ 17:46:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 17:46:04 DEBUG { 17:46:04 DEBUG ----- output ----- 17:46:04 DEBUG "metric": { 17:46:04 DEBUG { 17:46:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:46:04 DEBUG "status": "success", 17:46:04 DEBUG }, 17:46:04 DEBUG "data": { 17:46:04 DEBUG "resultType": "vector", 17:46:04 DEBUG "result": [ 17:46:04 DEBUG { 17:46:04 DEBUG "metric": { 17:46:04 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:46:04 DEBUG }, 17:46:04 DEBUG "value": [ 17:46:04 DEBUG "value": [ 17:46:04 DEBUG 17:46:04 DEBUG 17:46:04 DEBUG 17:46:04 DEBUG 17:46:04 DEBUG 17:46:04 DEBUG 17:46:04 DEBUG [...] 17:46:04 DEBUG 17:46:04 DEBUG [...] 17:46:04 DEBUG 17:46:04 DEBUG 17:46:04 DEBUG 17:46:04 DEBUG 17:46:04 DEBUG "pod": "ds-idrepo-2" 17:46:04 DEBUG 17:46:04 DEBUG }, 17:46:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:46:04 DEBUG "value": [ 17:46:04 DEBUG }, 17:46:04 DEBUG 1713545164, 17:46:04 DEBUG "value": [ 17:46:04 DEBUG "149543495.41016835" 17:46:04 DEBUG 1713545164, 17:46:04 DEBUG ] 17:46:04 DEBUG "2490.2" 17:46:04 DEBUG } 17:46:04 DEBUG ] 17:46:04 DEBUG } 17:46:04 DEBUG ] 17:46:04 DEBUG ] 17:46:04 DEBUG } 17:46:04 DEBUG } 17:46:04 DEBUG } 17:46:04 DEBUG } 17:46:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545165" 17:46:05 INFO [http_cmd]: http status code OK 17:46:05 DEBUG --- status code --- 17:46:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:05 DEBUG --- http response --- 17:46:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:46:05 DEBUG ----- output ----- 17:46:05 DEBUG { 17:46:05 DEBUG "status": "success", 17:46:05 DEBUG "data": { 17:46:05 DEBUG "resultType": "vector", 17:46:05 DEBUG "result": [ 17:46:05 DEBUG { 17:46:05 DEBUG "metric": { 17:46:05 DEBUG "instance": "10.142.0.107:9100" 17:46:05 DEBUG }, 17:46:05 DEBUG "value": [ 17:46:05 DEBUG 17:46:05 DEBUG 17:46:05 DEBUG 17:46:05 DEBUG [...] 17:46:05 DEBUG 17:46:05 DEBUG 17:46:05 DEBUG 17:46:05 DEBUG "instance": "10.142.0.43:9100" 17:46:05 DEBUG }, 17:46:05 DEBUG "value": [ 17:46:05 DEBUG 1713545165, 17:46:05 DEBUG "0.7036666666666728" 17:46:05 DEBUG ] 17:46:05 DEBUG } 17:46:05 DEBUG ] 17:46:05 DEBUG } 17:46:05 DEBUG } 17:46:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545165" 17:46:05 INFO [http_cmd]: http status code OK 17:46:05 DEBUG --- status code --- 17:46:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:05 DEBUG --- http response --- 17:46:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:46:05 DEBUG ----- output ----- 17:46:05 DEBUG { 17:46:05 DEBUG "status": "success", 17:46:05 DEBUG "data": { 17:46:05 DEBUG "resultType": "vector", 17:46:05 DEBUG "result": [ 17:46:05 DEBUG { 17:46:05 DEBUG "metric": { 17:46:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:46:05 DEBUG }, 17:46:05 DEBUG "value": [ 17:46:05 DEBUG 17:46:05 DEBUG 17:46:05 DEBUG 17:46:05 DEBUG [...] 17:46:05 DEBUG 17:46:05 DEBUG 17:46:05 DEBUG 17:46:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:46:05 DEBUG }, 17:46:05 DEBUG "value": [ 17:46:05 DEBUG 1713545165, 17:46:05 DEBUG "4206445.5" 17:46:05 DEBUG ] 17:46:05 DEBUG } 17:46:05 DEBUG ] 17:46:05 DEBUG } 17:46:05 DEBUG } 17:46:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545166" 17:46:06 INFO [http_cmd]: http status code OK 17:46:06 DEBUG --- status code --- 17:46:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:06 DEBUG --- http response --- 17:46:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:46:06 DEBUG ----- output ----- 17:46:06 DEBUG { 17:46:06 DEBUG "status": "success", 17:46:06 DEBUG "data": { 17:46:06 DEBUG "resultType": "vector", 17:46:06 DEBUG "result": [ 17:46:06 DEBUG { 17:46:06 DEBUG "metric": { 17:46:06 DEBUG "instance": "10.142.0.107:9100" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG [...] 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG "instance": "10.142.0.43:9100" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 1713545166, 17:46:06 DEBUG "1765421929472" 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG } 17:46:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545166" 17:46:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545166" 17:46:06 INFO [http_cmd]: http status code OK 17:46:06 DEBUG --- status code --- 17:46:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:06 DEBUG --- http response --- 17:46:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:46:06 DEBUG ----- output ----- 17:46:06 DEBUG { 17:46:06 DEBUG "status": "success", 17:46:06 DEBUG "data": { 17:46:06 DEBUG "resultType": "vector", 17:46:06 DEBUG "result": [ 17:46:06 DEBUG { 17:46:06 DEBUG "metric": { 17:46:06 DEBUG "pod": "ds-idrepo-0" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG [...] 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG "pod": "ds-idrepo-2" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 1713545166, 17:46:06 DEBUG "0" 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG } 17:46:06 INFO [http_cmd]: http status code OK 17:46:06 DEBUG --- status code --- 17:46:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:06 DEBUG --- http response --- 17:46:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:46:06 DEBUG ----- output ----- 17:46:06 DEBUG { 17:46:06 DEBUG "status": "success", 17:46:06 DEBUG "data": { 17:46:06 DEBUG "resultType": "vector", 17:46:06 DEBUG "result": [ 17:46:06 DEBUG { 17:46:06 DEBUG "metric": { 17:46:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG [...] 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 1713545166, 17:46:06 DEBUG "1839910912" 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG } 17:46:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545166" 17:46:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545166" 17:46:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545166" 17:46:06 INFO [http_cmd]: http status code OK 17:46:06 DEBUG --- status code --- 17:46:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:06 DEBUG --- http response --- 17:46:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:46:06 DEBUG ----- output ----- 17:46:06 DEBUG { 17:46:06 DEBUG "status": "success", 17:46:06 DEBUG "data": { 17:46:06 DEBUG "resultType": "vector", 17:46:06 DEBUG "result": [ 17:46:06 DEBUG { 17:46:06 DEBUG "metric": { 17:46:06 DEBUG "pod": "ds-cts-0" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG [...] 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG "pod": "ds-cts-2" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 1713545166, 17:46:06 DEBUG "0.35576666666666673" 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG } 17:46:06 INFO [http_cmd]: http status code OK 17:46:06 DEBUG --- status code --- 17:46:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:06 DEBUG --- http response --- 17:46:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:46:06 DEBUG ----- output ----- 17:46:06 DEBUG { 17:46:06 DEBUG "status": "success", 17:46:06 DEBUG "data": { 17:46:06 DEBUG "resultType": "vector", 17:46:06 DEBUG "result": [ 17:46:06 DEBUG { 17:46:06 DEBUG "metric": { 17:46:06 DEBUG "pod": "ds-idrepo-0" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG [...] 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG "pod": "ds-idrepo-2" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 1713545166, 17:46:06 DEBUG "0" 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG } 17:46:06 INFO [http_cmd]: http status code OK 17:46:06 DEBUG --- status code --- 17:46:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:06 DEBUG --- http response --- 17:46:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:46:06 DEBUG ----- output ----- 17:46:06 DEBUG { 17:46:06 DEBUG "status": "success", 17:46:06 DEBUG "data": { 17:46:06 DEBUG "resultType": "vector", 17:46:06 DEBUG "result": [ 17:46:06 DEBUG { 17:46:06 DEBUG "metric": { 17:46:06 DEBUG "pod": "ds-cts-0" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG [...] 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG "pod": "ds-cts-2" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 1713545166, 17:46:06 DEBUG "29.733333333333334" 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG } 17:46:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545166" 17:46:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545166" 17:46:06 INFO [http_cmd]: http status code OK 17:46:06 DEBUG --- status code --- 17:46:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:06 DEBUG --- http response --- 17:46:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:46:06 DEBUG ----- output ----- 17:46:06 DEBUG { 17:46:06 DEBUG "status": "success", 17:46:06 DEBUG "data": { 17:46:06 DEBUG "resultType": "vector", 17:46:06 DEBUG "result": [ 17:46:06 DEBUG { 17:46:06 INFO [http_cmd]: http status code OK 17:46:06 DEBUG "metric": { 17:46:06 DEBUG --- status code --- 17:46:06 DEBUG "instance": "10.142.0.107:9100" 17:46:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:06 DEBUG }, 17:46:06 DEBUG --- http response --- 17:46:06 DEBUG "value": [ 17:46:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:46:06 DEBUG 17:46:06 DEBUG ----- output ----- 17:46:06 DEBUG 17:46:06 DEBUG { 17:46:06 DEBUG 17:46:06 DEBUG "status": "success", 17:46:06 DEBUG [...] 17:46:06 DEBUG "data": { 17:46:06 DEBUG 17:46:06 DEBUG "resultType": "vector", 17:46:06 DEBUG 17:46:06 DEBUG "result": [ 17:46:06 DEBUG 17:46:06 DEBUG { 17:46:06 DEBUG "instance": "10.142.0.43:9100" 17:46:06 DEBUG "metric": { 17:46:06 DEBUG }, 17:46:06 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:46:06 DEBUG "value": [ 17:46:06 DEBUG }, 17:46:06 DEBUG 1713545166, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG "74688850.36666667" 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG ] 17:46:06 DEBUG 17:46:06 DEBUG } 17:46:06 DEBUG } 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG [...] 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 1713545166, 17:46:06 DEBUG "0" 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG } 17:46:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545166" 17:46:06 INFO [http_cmd]: http status code OK 17:46:06 DEBUG --- status code --- 17:46:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:06 DEBUG --- http response --- 17:46:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:46:06 DEBUG ----- output ----- 17:46:06 DEBUG { 17:46:06 DEBUG "status": "success", 17:46:06 DEBUG "data": { 17:46:06 DEBUG "resultType": "vector", 17:46:06 DEBUG "result": [ 17:46:06 DEBUG { 17:46:06 DEBUG "metric": { 17:46:06 DEBUG "instance": "10.142.0.107:9100" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG [...] 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG 17:46:06 DEBUG "instance": "10.142.0.43:9100" 17:46:06 DEBUG }, 17:46:06 DEBUG "value": [ 17:46:06 DEBUG 1713545166, 17:46:06 DEBUG "74682500.80000001" 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG ] 17:46:06 DEBUG } 17:46:06 DEBUG } 17:46:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545167" 17:46:07 INFO [http_cmd]: http status code OK 17:46:07 DEBUG --- status code --- 17:46:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:07 DEBUG --- http response --- 17:46:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:46:07 DEBUG ----- output ----- 17:46:07 DEBUG { 17:46:07 DEBUG "status": "success", 17:46:07 DEBUG "data": { 17:46:07 DEBUG "resultType": "vector", 17:46:07 DEBUG "result": [ 17:46:07 DEBUG { 17:46:07 DEBUG "metric": { 17:46:07 DEBUG "pod": "ds-idrepo-0" 17:46:07 DEBUG }, 17:46:07 DEBUG "value": [ 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG [...] 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG "pod": "ds-idrepo-2" 17:46:07 DEBUG }, 17:46:07 DEBUG "value": [ 17:46:07 DEBUG 1713545167, 17:46:07 DEBUG "0" 17:46:07 DEBUG ] 17:46:07 DEBUG } 17:46:07 DEBUG ] 17:46:07 DEBUG } 17:46:07 DEBUG } 17:46:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545167" 17:46:07 INFO [http_cmd]: http status code OK 17:46:07 DEBUG --- status code --- 17:46:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:07 DEBUG --- http response --- 17:46:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:46:07 DEBUG ----- output ----- 17:46:07 DEBUG { 17:46:07 DEBUG "status": "success", 17:46:07 DEBUG "data": { 17:46:07 DEBUG "resultType": "vector", 17:46:07 DEBUG "result": [ 17:46:07 DEBUG { 17:46:07 DEBUG "metric": { 17:46:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:46:07 DEBUG }, 17:46:07 DEBUG "value": [ 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG [...] 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:46:07 DEBUG }, 17:46:07 DEBUG "value": [ 17:46:07 DEBUG 1713545167, 17:46:07 DEBUG "2488.8333333333335" 17:46:07 DEBUG ] 17:46:07 DEBUG } 17:46:07 DEBUG ] 17:46:07 DEBUG } 17:46:07 DEBUG } 17:46:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545167" 17:46:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545167" 17:46:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545167" 17:46:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545167" 17:46:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545167" 17:46:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545167" 17:46:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545167" 17:46:07 INFO [http_cmd]: http status code OK 17:46:07 DEBUG --- status code --- 17:46:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:07 DEBUG --- http response --- 17:46:07 INFO [http_cmd]: http status code OK 17:46:07 DEBUG --- status code --- 17:46:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:07 DEBUG --- http response --- 17:46:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:46:07 DEBUG ----- output ----- 17:46:07 DEBUG { 17:46:07 DEBUG "status": "success", 17:46:07 DEBUG "data": { 17:46:07 DEBUG "resultType": "vector", 17:46:07 DEBUG "result": [ 17:46:07 DEBUG { 17:46:07 INFO [http_cmd]: http status code OK 17:46:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:46:07 DEBUG "metric": { 17:46:07 INFO [http_cmd]: http status code OK 17:46:07 DEBUG --- status code --- 17:46:07 DEBUG ----- output ----- 17:46:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:46:07 INFO [http_cmd]: http status code OK 17:46:07 DEBUG --- status code --- 17:46:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:07 INFO [http_cmd]: http status code OK 17:46:07 DEBUG { 17:46:07 DEBUG }, 17:46:07 DEBUG --- status code --- 17:46:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:07 DEBUG --- http response --- 17:46:07 DEBUG --- status code --- 17:46:07 DEBUG "status": "success", 17:46:07 DEBUG "value": [ 17:46:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:07 DEBUG --- http response --- 17:46:07 INFO [http_cmd]: http status code OK 17:46:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:46:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:07 DEBUG "data": { 17:46:07 DEBUG 17:46:07 DEBUG --- http response --- 17:46:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:46:07 DEBUG --- status code --- 17:46:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:46:07 DEBUG --- http response --- 17:46:07 DEBUG "resultType": "vector", 17:46:07 DEBUG 17:46:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:46:07 DEBUG ----- output ----- 17:46:07 DEBUG --- http response --- 17:46:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 17:46:07 DEBUG "result": [ 17:46:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:46:07 DEBUG 17:46:07 DEBUG ----- output ----- 17:46:07 DEBUG { 17:46:07 DEBUG ----- output ----- 17:46:07 DEBUG ----- output ----- 17:46:07 DEBUG { 17:46:07 DEBUG ----- output ----- 17:46:07 DEBUG [...] 17:46:07 DEBUG { 17:46:07 DEBUG "status": "success", 17:46:07 DEBUG { 17:46:07 DEBUG { 17:46:07 DEBUG "metric": { 17:46:07 DEBUG { 17:46:07 DEBUG 17:46:07 DEBUG "status": "success", 17:46:07 DEBUG "data": { 17:46:07 DEBUG "status": "success", 17:46:07 DEBUG "status": "success", 17:46:07 DEBUG "pod": "ds-idrepo-0" 17:46:07 DEBUG "status": "success", 17:46:07 DEBUG 17:46:07 DEBUG "data": { 17:46:07 DEBUG "resultType": "vector", 17:46:07 DEBUG "data": { 17:46:07 DEBUG "data": { 17:46:07 DEBUG }, 17:46:07 DEBUG "data": { 17:46:07 DEBUG 17:46:07 DEBUG "resultType": "vector", 17:46:07 DEBUG "result": [ 17:46:07 DEBUG "resultType": "vector", 17:46:07 DEBUG "resultType": "vector", 17:46:07 DEBUG "value": [ 17:46:07 DEBUG "resultType": "vector", 17:46:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:46:07 DEBUG "result": [ 17:46:07 DEBUG { 17:46:07 DEBUG "result": [ 17:46:07 DEBUG "result": [ 17:46:07 DEBUG 17:46:07 DEBUG "result": [ 17:46:07 DEBUG }, 17:46:07 DEBUG { 17:46:07 DEBUG "metric": { 17:46:07 DEBUG { 17:46:07 DEBUG { 17:46:07 DEBUG 17:46:07 DEBUG { 17:46:07 DEBUG "value": [ 17:46:07 DEBUG "metric": { 17:46:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:46:07 DEBUG "metric": { 17:46:07 DEBUG "metric": { 17:46:07 DEBUG 17:46:07 DEBUG "metric": { 17:46:07 DEBUG 1713545167, 17:46:07 DEBUG "instance": "10.142.0.107:9100" 17:46:07 DEBUG }, 17:46:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:46:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:46:07 DEBUG [...] 17:46:07 DEBUG "pod": "ds-cts-0" 17:46:07 DEBUG "0.023" 17:46:07 DEBUG }, 17:46:07 DEBUG "value": [ 17:46:07 DEBUG }, 17:46:07 DEBUG }, 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG ] 17:46:07 DEBUG "value": [ 17:46:07 DEBUG 17:46:07 DEBUG "value": [ 17:46:07 DEBUG "value": [ 17:46:07 DEBUG 17:46:07 DEBUG } 17:46:07 DEBUG }, 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG "pod": "ds-idrepo-2" 17:46:07 DEBUG ] 17:46:07 DEBUG "value": [ 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG } 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG [...] 17:46:07 DEBUG 17:46:07 DEBUG [...] 17:46:07 DEBUG }, 17:46:07 DEBUG 17:46:07 DEBUG [...] 17:46:07 DEBUG 17:46:07 DEBUG [...] 17:46:07 DEBUG } 17:46:07 DEBUG 17:46:07 DEBUG "value": [ 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 1713545167, 17:46:07 DEBUG 17:46:07 DEBUG [...] 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG "0" 17:46:07 DEBUG 17:46:07 DEBUG 17:46:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:46:07 DEBUG 17:46:07 DEBUG "pod": "ds-idrepo-2" 17:46:07 DEBUG ] 17:46:07 DEBUG "pod": "ds-cts-2" 17:46:07 DEBUG 17:46:07 DEBUG }, 17:46:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:46:07 DEBUG }, 17:46:07 DEBUG } 17:46:07 DEBUG }, 17:46:07 DEBUG 17:46:07 DEBUG "value": [ 17:46:07 DEBUG }, 17:46:07 DEBUG "value": [ 17:46:07 DEBUG ] 17:46:07 DEBUG "value": [ 17:46:07 DEBUG "instance": "10.142.0.43:9100" 17:46:07 DEBUG 1713545167, 17:46:07 DEBUG "value": [ 17:46:07 DEBUG 1713545167, 17:46:07 DEBUG } 17:46:07 DEBUG 1713545167, 17:46:07 DEBUG }, 17:46:07 DEBUG "8.04005670483627" 17:46:07 DEBUG 1713545167, 17:46:07 DEBUG "2.989035119557871" 17:46:07 DEBUG } 17:46:07 DEBUG "233.66666666666666" 17:46:07 DEBUG "value": [ 17:46:07 DEBUG ] 17:46:07 DEBUG "2490.766666666667" 17:46:07 DEBUG ] 17:46:07 DEBUG ] 17:46:07 DEBUG 1713545167, 17:46:07 DEBUG } 17:46:07 DEBUG ] 17:46:07 DEBUG } 17:46:07 DEBUG } 17:46:07 DEBUG "0.9420666666667538" 17:46:07 DEBUG ] 17:46:07 DEBUG } 17:46:07 DEBUG ] 17:46:07 DEBUG ] 17:46:07 DEBUG ] 17:46:07 DEBUG } 17:46:07 DEBUG ] 17:46:07 DEBUG } 17:46:07 DEBUG } 17:46:07 DEBUG } 17:46:07 DEBUG } 17:46:07 DEBUG } 17:46:07 DEBUG } 17:46:07 DEBUG } 17:46:07 DEBUG } 17:46:07 DEBUG ] 17:46:07 DEBUG } 17:46:07 DEBUG } 17:46:08 INFO 17:46:08 INFO [loop_until]: kubectl --namespace=xlou top pods 17:46:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:46:08 INFO [loop_until]: OK (rc = 0) 17:46:08 DEBUG --- stdout --- 17:46:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17329m 9016Mi am-6fd554f8f7-btd4s 17476m 9062Mi am-6fd554f8f7-wxcsj 19236m 9052Mi ds-cts-0 20838m 12925Mi ds-cts-1 21713m 13197Mi ds-cts-2 21479m 12988Mi ds-idrepo-0 6900m 3424Mi ds-idrepo-1 6951m 3395Mi ds-idrepo-2 7008m 3453Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8082m 1663Mi 17:46:08 DEBUG --- stderr --- 17:46:08 DEBUG 17:46:08 INFO 17:46:08 INFO [loop_until]: kubectl --namespace=xlou top node 17:46:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:46:08 INFO [loop_until]: OK (rc = 0) 17:46:08 DEBUG --- stdout --- 17:46:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6171Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7447m 24% 7007Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 49m 0% 2682Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17606m 58% 12315Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 20900m 70% 15914Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5238Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17568m 58% 11971Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 17005m 56% 3072Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19526m 65% 11289Mi 10% gke-xlou-cdm-ds-0f601751-j488 7615m 25% 6689Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21049m 70% 18007Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 7145m 23% 7229Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21293m 71% 15403Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8191m 51% 5400Mi 9% 17:46:08 DEBUG --- stderr --- 17:46:08 DEBUG 17:47:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545224" 17:47:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545224" 17:47:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545224" 17:47:04 INFO [http_cmd]: http status code OK 17:47:04 INFO [http_cmd]: http status code OK 17:47:04 DEBUG --- status code --- 17:47:04 ERROR [http_cmd]: ERROR 17:47:04 DEBUG --- status code --- 17:47:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:04 DEBUG --- http response --- 17:47:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:04 ERROR --- status code --- 17:47:04 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:47:04 DEBUG --- http response --- 17:47:04 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:47:04 ERROR --- http response --- 17:47:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:47:04 DEBUG ----- output ----- 17:47:04 DEBUG { 17:47:04 DEBUG "status": "success", 17:47:04 DEBUG "data": { 17:47:04 DEBUG "resultType": "vector", 17:47:04 DEBUG "result": [ 17:47:04 DEBUG { 17:47:04 DEBUG "metric": { 17:47:04 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:47:04 DEBUG }, 17:47:04 DEBUG "value": [ 17:47:04 DEBUG 17:47:04 DEBUG 17:47:04 DEBUG 17:47:04 DEBUG [...] 17:47:04 DEBUG 17:47:04 DEBUG 17:47:04 DEBUG 17:47:04 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:47:04 DEBUG }, 17:47:04 DEBUG "value": [ 17:47:04 DEBUG 1713545224, 17:47:04 DEBUG "2304.8333333333335" 17:47:04 DEBUG ] 17:47:04 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:47:04 DEBUG } 17:47:04 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:47:04 DEBUG ] 17:47:04 DEBUG } 17:47:04 DEBUG } 17:47:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545225" 17:47:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545225" 17:47:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545225" 17:47:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545225" 17:47:05 INFO [http_cmd]: http status code OK 17:47:05 INFO [http_cmd]: http status code OK 17:47:05 DEBUG --- status code --- 17:47:05 DEBUG --- status code --- 17:47:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:05 DEBUG --- http response --- 17:47:05 DEBUG --- http response --- 17:47:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:47:05 DEBUG ----- output ----- 17:47:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:47:05 DEBUG { 17:47:05 INFO [http_cmd]: http status code OK 17:47:05 DEBUG ----- output ----- 17:47:05 DEBUG "status": "success", 17:47:05 DEBUG --- status code --- 17:47:05 DEBUG { 17:47:05 DEBUG "data": { 17:47:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:05 DEBUG "status": "success", 17:47:05 DEBUG "resultType": "vector", 17:47:05 DEBUG --- http response --- 17:47:05 DEBUG "data": { 17:47:05 DEBUG "result": [ 17:47:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:47:05 DEBUG "resultType": "vector", 17:47:05 DEBUG { 17:47:05 DEBUG ----- output ----- 17:47:05 DEBUG "result": [ 17:47:05 DEBUG "metric": { 17:47:05 DEBUG { 17:47:05 DEBUG { 17:47:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:47:05 DEBUG "status": "success", 17:47:05 DEBUG "metric": { 17:47:05 DEBUG }, 17:47:05 DEBUG "data": { 17:47:05 DEBUG "pod": "ds-idrepo-0" 17:47:05 DEBUG "value": [ 17:47:05 DEBUG "resultType": "vector", 17:47:05 DEBUG }, 17:47:05 DEBUG 17:47:05 DEBUG "result": [ 17:47:05 DEBUG "value": [ 17:47:05 DEBUG 17:47:05 DEBUG { 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG "metric": { 17:47:05 DEBUG 17:47:05 DEBUG [...] 17:47:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG }, 17:47:05 DEBUG [...] 17:47:05 DEBUG 17:47:05 DEBUG "value": [ 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:47:05 DEBUG 17:47:05 INFO [http_cmd]: http status code OK 17:47:05 DEBUG 17:47:05 DEBUG }, 17:47:05 DEBUG 17:47:05 DEBUG --- status code --- 17:47:05 DEBUG "pod": "ds-cts-2" 17:47:05 DEBUG "value": [ 17:47:05 DEBUG [...] 17:47:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:05 DEBUG }, 17:47:05 DEBUG 1713545225, 17:47:05 DEBUG 17:47:05 DEBUG --- http response --- 17:47:05 DEBUG "value": [ 17:47:05 DEBUG "0" 17:47:05 DEBUG 17:47:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:47:05 DEBUG 1713545225, 17:47:05 DEBUG "0" 17:47:05 DEBUG 17:47:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:47:05 DEBUG ] 17:47:05 DEBUG }, 17:47:05 DEBUG ----- output ----- 17:47:05 DEBUG } 17:47:05 DEBUG "value": [ 17:47:05 DEBUG { 17:47:05 DEBUG "status": "success", 17:47:05 DEBUG "data": { 17:47:05 DEBUG 1713545225, 17:47:05 DEBUG "resultType": "vector", 17:47:05 DEBUG ] 17:47:05 DEBUG "2488.9666666666667" 17:47:05 DEBUG "result": [ 17:47:05 DEBUG ] 17:47:05 DEBUG } 17:47:05 DEBUG ] 17:47:05 DEBUG { 17:47:05 DEBUG "metric": { 17:47:05 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:47:05 DEBUG } 17:47:05 DEBUG }, 17:47:05 DEBUG ] 17:47:05 DEBUG } 17:47:05 DEBUG ] 17:47:05 DEBUG "value": [ 17:47:05 DEBUG } 17:47:05 DEBUG } 17:47:05 DEBUG } 17:47:05 DEBUG 17:47:05 DEBUG } 17:47:05 DEBUG } 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG [...] 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:47:05 DEBUG }, 17:47:05 DEBUG "value": [ 17:47:05 DEBUG 1713545225, 17:47:05 DEBUG "0" 17:47:05 DEBUG ] 17:47:05 DEBUG } 17:47:05 DEBUG ] 17:47:05 DEBUG } 17:47:05 DEBUG } 17:47:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545225" 17:47:05 INFO [http_cmd]: http status code OK 17:47:05 DEBUG --- status code --- 17:47:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:05 DEBUG --- http response --- 17:47:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:47:05 DEBUG ----- output ----- 17:47:05 DEBUG { 17:47:05 DEBUG "status": "success", 17:47:05 DEBUG "data": { 17:47:05 DEBUG "resultType": "vector", 17:47:05 DEBUG "result": [ 17:47:05 DEBUG { 17:47:05 DEBUG "metric": { 17:47:05 DEBUG "instance": "10.142.0.107:9100" 17:47:05 DEBUG }, 17:47:05 DEBUG "value": [ 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG [...] 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG "instance": "10.142.0.43:9100" 17:47:05 DEBUG }, 17:47:05 DEBUG "value": [ 17:47:05 DEBUG 1713545225, 17:47:05 DEBUG "0.6730000000000036" 17:47:05 DEBUG ] 17:47:05 DEBUG } 17:47:05 DEBUG ] 17:47:05 DEBUG } 17:47:05 DEBUG } 17:47:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545225" 17:47:05 INFO [http_cmd]: http status code OK 17:47:05 DEBUG --- status code --- 17:47:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:05 DEBUG --- http response --- 17:47:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:47:05 DEBUG ----- output ----- 17:47:05 DEBUG { 17:47:05 DEBUG "status": "success", 17:47:05 DEBUG "data": { 17:47:05 DEBUG "resultType": "vector", 17:47:05 DEBUG "result": [ 17:47:05 DEBUG { 17:47:05 DEBUG "metric": { 17:47:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:47:05 DEBUG }, 17:47:05 DEBUG "value": [ 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG [...] 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG 17:47:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:47:05 DEBUG }, 17:47:05 DEBUG "value": [ 17:47:05 DEBUG 1713545225, 17:47:05 DEBUG "4215826.5" 17:47:05 DEBUG ] 17:47:05 DEBUG } 17:47:05 DEBUG ] 17:47:05 DEBUG } 17:47:05 DEBUG } 17:47:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545226" 17:47:06 INFO [http_cmd]: http status code OK 17:47:06 DEBUG --- status code --- 17:47:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:06 DEBUG --- http response --- 17:47:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:47:06 DEBUG ----- output ----- 17:47:06 DEBUG { 17:47:06 DEBUG "status": "success", 17:47:06 DEBUG "data": { 17:47:06 DEBUG "resultType": "vector", 17:47:06 DEBUG "result": [ 17:47:06 DEBUG { 17:47:06 DEBUG "metric": { 17:47:06 DEBUG "instance": "10.142.0.107:9100" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG [...] 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG "instance": "10.142.0.43:9100" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 1713545226, 17:47:06 DEBUG "1770007811072" 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG } 17:47:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545226" 17:47:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545226" 17:47:06 INFO [http_cmd]: http status code OK 17:47:06 DEBUG --- status code --- 17:47:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:06 DEBUG --- http response --- 17:47:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:47:06 DEBUG ----- output ----- 17:47:06 DEBUG { 17:47:06 DEBUG "status": "success", 17:47:06 DEBUG "data": { 17:47:06 DEBUG "resultType": "vector", 17:47:06 DEBUG "result": [ 17:47:06 DEBUG { 17:47:06 DEBUG "metric": { 17:47:06 DEBUG "pod": "ds-idrepo-0" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG [...] 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG "pod": "ds-idrepo-2" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 1713545226, 17:47:06 DEBUG "0" 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG } 17:47:06 INFO [http_cmd]: http status code OK 17:47:06 DEBUG --- status code --- 17:47:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:06 DEBUG --- http response --- 17:47:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:47:06 DEBUG ----- output ----- 17:47:06 DEBUG { 17:47:06 DEBUG "status": "success", 17:47:06 DEBUG "data": { 17:47:06 DEBUG "resultType": "vector", 17:47:06 DEBUG "result": [ 17:47:06 DEBUG { 17:47:06 DEBUG "metric": { 17:47:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG [...] 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 1713545226, 17:47:06 DEBUG "1845215232" 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG } 17:47:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545226" 17:47:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545226" 17:47:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545226" 17:47:06 INFO [http_cmd]: http status code OK 17:47:06 DEBUG --- status code --- 17:47:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:06 DEBUG --- http response --- 17:47:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:47:06 DEBUG ----- output ----- 17:47:06 DEBUG { 17:47:06 DEBUG "status": "success", 17:47:06 DEBUG "data": { 17:47:06 DEBUG "resultType": "vector", 17:47:06 DEBUG "result": [ 17:47:06 DEBUG { 17:47:06 DEBUG "metric": { 17:47:06 DEBUG "pod": "ds-cts-0" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG [...] 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG "pod": "ds-cts-2" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 1713545226, 17:47:06 DEBUG "0.362766666666667" 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG } 17:47:06 INFO [http_cmd]: http status code OK 17:47:06 INFO [http_cmd]: http status code OK 17:47:06 DEBUG --- status code --- 17:47:06 DEBUG --- status code --- 17:47:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:06 DEBUG --- http response --- 17:47:06 DEBUG --- http response --- 17:47:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:47:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:47:06 DEBUG ----- output ----- 17:47:06 DEBUG ----- output ----- 17:47:06 DEBUG { 17:47:06 DEBUG { 17:47:06 DEBUG "status": "success", 17:47:06 DEBUG "status": "success", 17:47:06 DEBUG "data": { 17:47:06 DEBUG "data": { 17:47:06 DEBUG "resultType": "vector", 17:47:06 DEBUG "resultType": "vector", 17:47:06 DEBUG "result": [ 17:47:06 DEBUG "result": [ 17:47:06 DEBUG { 17:47:06 DEBUG { 17:47:06 DEBUG "metric": { 17:47:06 DEBUG "metric": { 17:47:06 DEBUG "pod": "ds-idrepo-0" 17:47:06 DEBUG "pod": "ds-cts-0" 17:47:06 DEBUG }, 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG [...] 17:47:06 DEBUG [...] 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG "pod": "ds-idrepo-2" 17:47:06 DEBUG "pod": "ds-cts-2" 17:47:06 DEBUG }, 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 1713545226, 17:47:06 DEBUG 1713545226, 17:47:06 DEBUG "0" 17:47:06 DEBUG "30.833333333333332" 17:47:06 DEBUG ] 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG } 17:47:06 DEBUG ] 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG } 17:47:06 DEBUG } 17:47:06 DEBUG } 17:47:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545226" 17:47:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545226" 17:47:06 INFO [http_cmd]: http status code OK 17:47:06 DEBUG --- status code --- 17:47:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:06 DEBUG --- http response --- 17:47:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:47:06 DEBUG ----- output ----- 17:47:06 DEBUG { 17:47:06 DEBUG "status": "success", 17:47:06 DEBUG "data": { 17:47:06 DEBUG "resultType": "vector", 17:47:06 DEBUG "result": [ 17:47:06 DEBUG { 17:47:06 DEBUG "metric": { 17:47:06 DEBUG "instance": "10.142.0.107:9100" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG [...] 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG "instance": "10.142.0.43:9100" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 1713545226, 17:47:06 DEBUG "74865634.6" 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG } 17:47:06 INFO [http_cmd]: http status code OK 17:47:06 DEBUG --- status code --- 17:47:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:06 DEBUG --- http response --- 17:47:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:47:06 DEBUG ----- output ----- 17:47:06 DEBUG { 17:47:06 DEBUG "status": "success", 17:47:06 DEBUG "data": { 17:47:06 DEBUG "resultType": "vector", 17:47:06 DEBUG "result": [ 17:47:06 DEBUG { 17:47:06 DEBUG "metric": { 17:47:06 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG [...] 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG "pod": "ds-idrepo-1" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 1713545226, 17:47:06 DEBUG "6190.07384509703" 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG } 17:47:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545226" 17:47:06 INFO [http_cmd]: http status code OK 17:47:06 DEBUG --- status code --- 17:47:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:06 DEBUG --- http response --- 17:47:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:47:06 DEBUG ----- output ----- 17:47:06 DEBUG { 17:47:06 DEBUG "status": "success", 17:47:06 DEBUG "data": { 17:47:06 DEBUG "resultType": "vector", 17:47:06 DEBUG "result": [ 17:47:06 DEBUG { 17:47:06 DEBUG "metric": { 17:47:06 DEBUG "instance": "10.142.0.107:9100" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG [...] 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG 17:47:06 DEBUG "instance": "10.142.0.43:9100" 17:47:06 DEBUG }, 17:47:06 DEBUG "value": [ 17:47:06 DEBUG 1713545226, 17:47:06 DEBUG "74858734.86666667" 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG ] 17:47:06 DEBUG } 17:47:06 DEBUG } 17:47:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545227" 17:47:07 INFO [http_cmd]: http status code OK 17:47:07 DEBUG --- status code --- 17:47:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:07 DEBUG --- http response --- 17:47:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 17:47:07 DEBUG ----- output ----- 17:47:07 DEBUG { 17:47:07 DEBUG "status": "success", 17:47:07 DEBUG "data": { 17:47:07 DEBUG "resultType": "vector", 17:47:07 DEBUG "result": [ 17:47:07 DEBUG { 17:47:07 DEBUG "metric": { 17:47:07 DEBUG "pod": "ds-idrepo-0" 17:47:07 DEBUG }, 17:47:07 DEBUG "value": [ 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG [...] 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG "pod": "ds-idrepo-1" 17:47:07 DEBUG }, 17:47:07 DEBUG "value": [ 17:47:07 DEBUG 1713545227, 17:47:07 DEBUG "0" 17:47:07 DEBUG ] 17:47:07 DEBUG } 17:47:07 DEBUG ] 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545227" 17:47:07 INFO [http_cmd]: http status code OK 17:47:07 DEBUG --- status code --- 17:47:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:07 DEBUG --- http response --- 17:47:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:47:07 DEBUG ----- output ----- 17:47:07 DEBUG { 17:47:07 DEBUG "status": "success", 17:47:07 DEBUG "data": { 17:47:07 DEBUG "resultType": "vector", 17:47:07 DEBUG "result": [ 17:47:07 DEBUG { 17:47:07 DEBUG "metric": { 17:47:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:47:07 DEBUG }, 17:47:07 DEBUG "value": [ 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG [...] 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:47:07 DEBUG }, 17:47:07 DEBUG "value": [ 17:47:07 DEBUG 1713545227, 17:47:07 DEBUG "2488.9" 17:47:07 DEBUG ] 17:47:07 DEBUG } 17:47:07 DEBUG ] 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545227" 17:47:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545227" 17:47:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545227" 17:47:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545227" 17:47:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545227" 17:47:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545227" 17:47:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545227" 17:47:07 INFO [http_cmd]: http status code OK 17:47:07 DEBUG --- status code --- 17:47:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:07 DEBUG --- http response --- 17:47:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:47:07 INFO [http_cmd]: http status code OK 17:47:07 DEBUG ----- output ----- 17:47:07 DEBUG --- status code --- 17:47:07 DEBUG { 17:47:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:07 INFO [http_cmd]: http status code OK 17:47:07 DEBUG --- status code --- 17:47:07 DEBUG "status": "success", 17:47:07 DEBUG --- http response --- 17:47:07 INFO [http_cmd]: http status code OK 17:47:07 INFO [http_cmd]: http status code OK 17:47:07 INFO [http_cmd]: http status code OK 17:47:07 DEBUG --- status code --- 17:47:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:47:07 DEBUG --- status code --- 17:47:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:07 DEBUG --- http response --- 17:47:07 DEBUG --- http response --- 17:47:07 DEBUG ----- output ----- 17:47:07 DEBUG --- status code --- 17:47:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:07 DEBUG "data": { 17:47:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:47:07 DEBUG { 17:47:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:47:07 DEBUG --- http response --- 17:47:07 DEBUG --- http response --- 17:47:07 DEBUG "resultType": "vector", 17:47:07 DEBUG ----- output ----- 17:47:07 DEBUG "status": "success", 17:47:07 DEBUG ----- output ----- 17:47:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:47:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:47:07 INFO [http_cmd]: http status code OK 17:47:07 DEBUG "result": [ 17:47:07 DEBUG { 17:47:07 DEBUG "data": { 17:47:07 DEBUG { 17:47:07 DEBUG ----- output ----- 17:47:07 DEBUG ----- output ----- 17:47:07 DEBUG --- status code --- 17:47:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:47:07 DEBUG --- http response --- 17:47:07 DEBUG "resultType": "vector", 17:47:07 DEBUG "status": "success", 17:47:07 DEBUG { 17:47:07 DEBUG { 17:47:07 DEBUG "status": "success", 17:47:07 DEBUG "data": { 17:47:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:47:07 DEBUG "result": [ 17:47:07 DEBUG "data": { 17:47:07 DEBUG "status": "success", 17:47:07 DEBUG "status": "success", 17:47:07 DEBUG "resultType": "vector", 17:47:07 DEBUG { 17:47:07 DEBUG ----- output ----- 17:47:07 DEBUG { 17:47:07 DEBUG "resultType": "vector", 17:47:07 DEBUG "data": { 17:47:07 DEBUG "data": { 17:47:07 DEBUG "result": [ 17:47:07 DEBUG "metric": { 17:47:07 DEBUG { 17:47:07 DEBUG "metric": { 17:47:07 DEBUG "result": [ 17:47:07 DEBUG "resultType": "vector", 17:47:07 DEBUG "resultType": "vector", 17:47:07 DEBUG { 17:47:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:47:07 DEBUG "status": "success", 17:47:07 DEBUG "pod": "ds-idrepo-0" 17:47:07 DEBUG { 17:47:07 DEBUG "result": [ 17:47:07 DEBUG "result": [ 17:47:07 DEBUG { 17:47:07 DEBUG }, 17:47:07 DEBUG "data": { 17:47:07 DEBUG }, 17:47:07 DEBUG "metric": { 17:47:07 DEBUG { 17:47:07 DEBUG "metric": { 17:47:07 DEBUG "value": [ 17:47:07 DEBUG "resultType": "vector", 17:47:07 DEBUG "value": [ 17:47:07 DEBUG "metric": { 17:47:07 DEBUG "pod": "ds-cts-0" 17:47:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:47:07 DEBUG "metric": { 17:47:07 DEBUG 17:47:07 DEBUG "result": [ 17:47:07 DEBUG 17:47:07 DEBUG "instance": "10.142.0.107:9100" 17:47:07 DEBUG }, 17:47:07 DEBUG }, 17:47:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:47:07 DEBUG 17:47:07 DEBUG { 17:47:07 DEBUG 17:47:07 DEBUG }, 17:47:07 DEBUG "value": [ 17:47:07 DEBUG "value": [ 17:47:07 DEBUG }, 17:47:07 DEBUG 17:47:07 DEBUG "metric": { 17:47:07 DEBUG 17:47:07 DEBUG "value": [ 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG "value": [ 17:47:07 DEBUG [...] 17:47:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:47:07 DEBUG [...] 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG }, 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG [...] 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG "value": [ 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG [...] 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG [...] 17:47:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:47:07 DEBUG 17:47:07 DEBUG "pod": "ds-idrepo-2" 17:47:07 DEBUG 17:47:07 DEBUG "pod": "ds-cts-2" 17:47:07 DEBUG [...] 17:47:07 DEBUG 17:47:07 DEBUG }, 17:47:07 DEBUG 17:47:07 DEBUG }, 17:47:07 DEBUG 17:47:07 DEBUG }, 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG "value": [ 17:47:07 DEBUG [...] 17:47:07 DEBUG "value": [ 17:47:07 DEBUG 17:47:07 DEBUG "value": [ 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 17:47:07 DEBUG 1713545227, 17:47:07 DEBUG "instance": "10.142.0.43:9100" 17:47:07 DEBUG 1713545227, 17:47:07 DEBUG 1713545227, 17:47:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:47:07 DEBUG }, 17:47:07 DEBUG 17:47:07 DEBUG "0" 17:47:07 DEBUG }, 17:47:07 DEBUG "238.06666666666666" 17:47:07 DEBUG "0.023" 17:47:07 DEBUG "value": [ 17:47:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:47:07 DEBUG 17:47:07 DEBUG ] 17:47:07 DEBUG "value": [ 17:47:07 DEBUG ] 17:47:07 DEBUG ] 17:47:07 DEBUG 1713545227, 17:47:07 DEBUG }, 17:47:07 DEBUG "pod": "idm-b85f9d978-txpwv" 17:47:07 DEBUG }, 17:47:07 DEBUG 1713545227, 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 DEBUG "8.036592786440552" 17:47:07 DEBUG "value": [ 17:47:07 DEBUG "value": [ 17:47:07 DEBUG } 17:47:07 DEBUG "0.9335333333332992" 17:47:07 DEBUG ] 17:47:07 DEBUG ] 17:47:07 DEBUG ] 17:47:07 DEBUG 1713545227, 17:47:07 DEBUG 1713545227, 17:47:07 DEBUG ] 17:47:07 DEBUG ] 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 DEBUG "2486.733333333333" 17:47:07 DEBUG "0" 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 DEBUG ] 17:47:07 DEBUG ] 17:47:07 DEBUG ] 17:47:07 DEBUG } 17:47:07 DEBUG ] 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 DEBUG ] 17:47:07 DEBUG } 17:47:07 DEBUG ] 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:07 DEBUG } 17:47:08 INFO 17:47:08 INFO [loop_until]: kubectl --namespace=xlou top pods 17:47:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:47:08 INFO [loop_until]: OK (rc = 0) 17:47:08 DEBUG --- stdout --- 17:47:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17310m 9016Mi am-6fd554f8f7-btd4s 18226m 9062Mi am-6fd554f8f7-wxcsj 19402m 9053Mi ds-cts-0 21497m 13282Mi ds-cts-1 20794m 13453Mi ds-cts-2 21506m 13276Mi ds-idrepo-0 6894m 3424Mi ds-idrepo-1 6954m 3394Mi ds-idrepo-2 7148m 3453Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1172Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7888m 1665Mi 17:47:08 DEBUG --- stderr --- 17:47:08 DEBUG 17:47:08 INFO 17:47:08 INFO [loop_until]: kubectl --namespace=xlou top node 17:47:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:47:08 INFO [loop_until]: OK (rc = 0) 17:47:08 DEBUG --- stdout --- 17:47:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6181Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7511m 25% 7004Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 50m 0% 2675Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17687m 59% 12313Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 20890m 69% 16305Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5240Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17631m 59% 11963Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 16967m 56% 3074Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19553m 65% 11294Mi 10% gke-xlou-cdm-ds-0f601751-j488 7645m 25% 6687Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21137m 70% 18218Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 7449m 24% 7236Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21440m 71% 15779Mi 14% gke-xlou-cdm-frontend-a8771548-i82n 8198m 51% 5397Mi 9% 17:47:08 DEBUG --- stderr --- 17:47:08 DEBUG 17:48:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545285" 17:48:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545285" 17:48:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545285" 17:48:05 INFO [http_cmd]: http status code OK 17:48:05 ERROR [http_cmd]: ERROR 17:48:05 INFO [http_cmd]: http status code OK 17:48:05 DEBUG --- status code --- 17:48:05 ERROR --- status code --- 17:48:05 DEBUG --- status code --- 17:48:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:05 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:48:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:05 DEBUG --- http response --- 17:48:05 ERROR --- http response --- 17:48:05 DEBUG --- http response --- 17:48:05 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:48:05 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:48:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:48:05 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:48:05 DEBUG ----- output ----- 17:48:05 DEBUG { 17:48:05 DEBUG "status": "success", 17:48:05 DEBUG "data": { 17:48:05 DEBUG "resultType": "vector", 17:48:05 DEBUG "result": [ 17:48:05 DEBUG { 17:48:05 DEBUG "metric": { 17:48:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:48:05 DEBUG }, 17:48:05 DEBUG "value": [ 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG [...] 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:48:05 DEBUG }, 17:48:05 DEBUG "value": [ 17:48:05 DEBUG 1713545285, 17:48:05 DEBUG "2324.9333333333334" 17:48:05 DEBUG ] 17:48:05 DEBUG } 17:48:05 DEBUG ] 17:48:05 DEBUG } 17:48:05 DEBUG } 17:48:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545285" 17:48:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545285" 17:48:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545285" 17:48:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545285" 17:48:05 INFO [http_cmd]: http status code OK 17:48:05 DEBUG --- status code --- 17:48:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:05 DEBUG --- http response --- 17:48:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:48:05 DEBUG ----- output ----- 17:48:05 DEBUG { 17:48:05 DEBUG "status": "success", 17:48:05 DEBUG "data": { 17:48:05 DEBUG "resultType": "vector", 17:48:05 DEBUG "result": [ 17:48:05 DEBUG { 17:48:05 DEBUG "metric": { 17:48:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:48:05 DEBUG }, 17:48:05 INFO [http_cmd]: http status code OK 17:48:05 INFO [http_cmd]: http status code OK 17:48:05 DEBUG "value": [ 17:48:05 DEBUG --- status code --- 17:48:05 DEBUG --- status code --- 17:48:05 DEBUG 17:48:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:05 DEBUG 17:48:05 DEBUG --- http response --- 17:48:05 DEBUG --- http response --- 17:48:05 DEBUG 17:48:05 INFO [http_cmd]: http status code OK 17:48:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:48:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:48:05 DEBUG [...] 17:48:05 DEBUG --- status code --- 17:48:05 DEBUG ----- output ----- 17:48:05 DEBUG ----- output ----- 17:48:05 DEBUG 17:48:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:05 DEBUG { 17:48:05 DEBUG { 17:48:05 DEBUG 17:48:05 DEBUG --- http response --- 17:48:05 DEBUG "status": "success", 17:48:05 DEBUG "status": "success", 17:48:05 DEBUG 17:48:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:48:05 DEBUG "data": { 17:48:05 DEBUG "data": { 17:48:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:48:05 DEBUG ----- output ----- 17:48:05 DEBUG "resultType": "vector", 17:48:05 DEBUG "resultType": "vector", 17:48:05 DEBUG }, 17:48:05 DEBUG { 17:48:05 DEBUG "result": [ 17:48:05 DEBUG { 17:48:05 DEBUG "value": [ 17:48:05 DEBUG "status": "success", 17:48:05 DEBUG "metric": { 17:48:05 DEBUG 1713545285, 17:48:05 DEBUG "result": [ 17:48:05 DEBUG "data": { 17:48:05 DEBUG "pod": "ds-idrepo-0" 17:48:05 DEBUG "0" 17:48:05 DEBUG { 17:48:05 DEBUG "resultType": "vector", 17:48:05 DEBUG }, 17:48:05 DEBUG ] 17:48:05 DEBUG "metric": { 17:48:05 DEBUG "result": [ 17:48:05 DEBUG "value": [ 17:48:05 DEBUG } 17:48:05 DEBUG "pod": "ds-idrepo-0" 17:48:05 DEBUG { 17:48:05 DEBUG 17:48:05 DEBUG ] 17:48:05 DEBUG }, 17:48:05 DEBUG "value": [ 17:48:05 DEBUG 17:48:05 DEBUG } 17:48:05 DEBUG "metric": { 17:48:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:48:05 DEBUG }, 17:48:05 DEBUG } 17:48:05 DEBUG "value": [ 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG [...] 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG [...] 17:48:05 DEBUG "pod": "ds-cts-2" 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG }, 17:48:05 DEBUG [...] 17:48:05 DEBUG 17:48:05 DEBUG "value": [ 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG 1713545285, 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG "0" 17:48:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:48:05 DEBUG "pod": "idm-b85f9d978-txpwv" 17:48:05 DEBUG ] 17:48:05 DEBUG }, 17:48:05 DEBUG }, 17:48:05 DEBUG } 17:48:05 DEBUG "value": [ 17:48:05 DEBUG 1713545285, 17:48:05 DEBUG ] 17:48:05 DEBUG "value": [ 17:48:05 DEBUG "2494.0333333333333" 17:48:05 DEBUG } 17:48:05 DEBUG 1713545285, 17:48:05 DEBUG ] 17:48:05 DEBUG } 17:48:05 DEBUG "108.49788524365671" 17:48:05 DEBUG } 17:48:05 DEBUG ] 17:48:05 DEBUG ] 17:48:05 DEBUG } 17:48:05 DEBUG } 17:48:05 DEBUG ] 17:48:05 DEBUG } 17:48:05 DEBUG } 17:48:05 DEBUG } 17:48:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545285" 17:48:05 INFO [http_cmd]: http status code OK 17:48:05 DEBUG --- status code --- 17:48:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:05 DEBUG --- http response --- 17:48:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:48:05 DEBUG ----- output ----- 17:48:05 DEBUG { 17:48:05 DEBUG "status": "success", 17:48:05 DEBUG "data": { 17:48:05 DEBUG "resultType": "vector", 17:48:05 DEBUG "result": [ 17:48:05 DEBUG { 17:48:05 DEBUG "metric": { 17:48:05 DEBUG "instance": "10.142.0.107:9100" 17:48:05 DEBUG }, 17:48:05 DEBUG "value": [ 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG [...] 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG "instance": "10.142.0.43:9100" 17:48:05 DEBUG }, 17:48:05 DEBUG "value": [ 17:48:05 DEBUG 1713545285, 17:48:05 DEBUG "0.7129999999999939" 17:48:05 DEBUG ] 17:48:05 DEBUG } 17:48:05 DEBUG ] 17:48:05 DEBUG } 17:48:05 DEBUG } 17:48:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545285" 17:48:05 INFO [http_cmd]: http status code OK 17:48:05 DEBUG --- status code --- 17:48:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:05 DEBUG --- http response --- 17:48:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:48:05 DEBUG ----- output ----- 17:48:05 DEBUG { 17:48:05 DEBUG "status": "success", 17:48:05 DEBUG "data": { 17:48:05 DEBUG "resultType": "vector", 17:48:05 DEBUG "result": [ 17:48:05 DEBUG { 17:48:05 DEBUG "metric": { 17:48:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:48:05 DEBUG }, 17:48:05 DEBUG "value": [ 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG [...] 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG 17:48:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:48:05 DEBUG }, 17:48:05 DEBUG "value": [ 17:48:05 DEBUG 1713545285, 17:48:05 DEBUG "4226765.5" 17:48:05 DEBUG ] 17:48:05 DEBUG } 17:48:05 DEBUG ] 17:48:05 DEBUG } 17:48:05 DEBUG } 17:48:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545286" 17:48:06 INFO [http_cmd]: http status code OK 17:48:06 DEBUG --- status code --- 17:48:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:06 DEBUG --- http response --- 17:48:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:48:06 DEBUG ----- output ----- 17:48:06 DEBUG { 17:48:06 DEBUG "status": "success", 17:48:06 DEBUG "data": { 17:48:06 DEBUG "resultType": "vector", 17:48:06 DEBUG "result": [ 17:48:06 DEBUG { 17:48:06 DEBUG "metric": { 17:48:06 DEBUG "instance": "10.142.0.107:9100" 17:48:06 DEBUG }, 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG [...] 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG "instance": "10.142.0.43:9100" 17:48:06 DEBUG }, 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 1713545286, 17:48:06 DEBUG "1774634320896" 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG } 17:48:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545286" 17:48:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545286" 17:48:06 INFO [http_cmd]: http status code OK 17:48:06 DEBUG --- status code --- 17:48:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:06 DEBUG --- http response --- 17:48:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:48:06 DEBUG ----- output ----- 17:48:06 DEBUG { 17:48:06 DEBUG "status": "success", 17:48:06 DEBUG "data": { 17:48:06 DEBUG "resultType": "vector", 17:48:06 DEBUG "result": [ 17:48:06 DEBUG { 17:48:06 DEBUG "metric": { 17:48:06 DEBUG "pod": "ds-idrepo-0" 17:48:06 DEBUG }, 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG [...] 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG "pod": "ds-idrepo-2" 17:48:06 DEBUG }, 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 1713545286, 17:48:06 DEBUG "0" 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG } 17:48:06 INFO [http_cmd]: http status code OK 17:48:06 DEBUG --- status code --- 17:48:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:06 DEBUG --- http response --- 17:48:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:48:06 DEBUG ----- output ----- 17:48:06 DEBUG { 17:48:06 DEBUG "status": "success", 17:48:06 DEBUG "data": { 17:48:06 DEBUG "resultType": "vector", 17:48:06 DEBUG "result": [ 17:48:06 DEBUG { 17:48:06 DEBUG "metric": { 17:48:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:48:06 DEBUG }, 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG [...] 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:48:06 DEBUG }, 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 1713545286, 17:48:06 DEBUG "1841590272" 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG } 17:48:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545286" 17:48:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545286" 17:48:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545286" 17:48:06 INFO [http_cmd]: http status code OK 17:48:06 DEBUG --- status code --- 17:48:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:06 DEBUG --- http response --- 17:48:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:48:06 DEBUG ----- output ----- 17:48:06 DEBUG { 17:48:06 DEBUG "status": "success", 17:48:06 DEBUG "data": { 17:48:06 DEBUG "resultType": "vector", 17:48:06 DEBUG "result": [ 17:48:06 DEBUG { 17:48:06 DEBUG "metric": { 17:48:06 DEBUG "pod": "ds-cts-0" 17:48:06 DEBUG }, 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG [...] 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG "pod": "ds-cts-2" 17:48:06 DEBUG }, 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 1713545286, 17:48:06 DEBUG "0.3325333333333333" 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG } 17:48:06 INFO [http_cmd]: http status code OK 17:48:06 DEBUG --- status code --- 17:48:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:06 DEBUG --- http response --- 17:48:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:48:06 DEBUG ----- output ----- 17:48:06 DEBUG { 17:48:06 DEBUG "status": "success", 17:48:06 DEBUG "data": { 17:48:06 DEBUG "resultType": "vector", 17:48:06 DEBUG "result": [ 17:48:06 DEBUG { 17:48:06 DEBUG "metric": { 17:48:06 DEBUG "pod": "ds-cts-0" 17:48:06 DEBUG }, 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG [...] 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG "pod": "ds-cts-2" 17:48:06 DEBUG }, 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 1713545286, 17:48:06 DEBUG "31.266666666666666" 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG } 17:48:06 INFO [http_cmd]: http status code OK 17:48:06 DEBUG --- status code --- 17:48:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:06 DEBUG --- http response --- 17:48:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:48:06 DEBUG ----- output ----- 17:48:06 DEBUG { 17:48:06 DEBUG "status": "success", 17:48:06 DEBUG "data": { 17:48:06 DEBUG "resultType": "vector", 17:48:06 DEBUG "result": [ 17:48:06 DEBUG { 17:48:06 DEBUG "metric": { 17:48:06 DEBUG "pod": "ds-idrepo-0" 17:48:06 DEBUG }, 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG [...] 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG "pod": "ds-idrepo-2" 17:48:06 DEBUG }, 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 1713545286, 17:48:06 DEBUG "0" 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG } 17:48:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545286" 17:48:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545286" 17:48:06 INFO [http_cmd]: http status code OK 17:48:06 INFO [http_cmd]: http status code OK 17:48:06 DEBUG --- status code --- 17:48:06 DEBUG --- status code --- 17:48:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:06 DEBUG --- http response --- 17:48:06 DEBUG --- http response --- 17:48:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:48:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:48:06 DEBUG ----- output ----- 17:48:06 DEBUG { 17:48:06 DEBUG "status": "success", 17:48:06 DEBUG ----- output ----- 17:48:06 DEBUG "data": { 17:48:06 DEBUG { 17:48:06 DEBUG "resultType": "vector", 17:48:06 DEBUG "status": "success", 17:48:06 DEBUG "result": [ 17:48:06 DEBUG "data": { 17:48:06 DEBUG { 17:48:06 DEBUG "resultType": "vector", 17:48:06 DEBUG "metric": { 17:48:06 DEBUG "result": [ 17:48:06 DEBUG "pod": "ds-idrepo-0" 17:48:06 DEBUG { 17:48:06 DEBUG }, 17:48:06 DEBUG "metric": { 17:48:06 DEBUG "value": [ 17:48:06 DEBUG "instance": "10.142.0.107:9100" 17:48:06 DEBUG 17:48:06 DEBUG }, 17:48:06 DEBUG 17:48:06 DEBUG "value": [ 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG [...] 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG [...] 17:48:06 DEBUG 17:48:06 DEBUG 17:48:06 DEBUG "pod": "idm-b85f9d978-txpwv" 17:48:06 DEBUG 17:48:06 DEBUG }, 17:48:06 DEBUG 17:48:06 DEBUG "value": [ 17:48:06 DEBUG "instance": "10.142.0.43:9100" 17:48:06 DEBUG 1713545286, 17:48:06 DEBUG }, 17:48:06 DEBUG "208.98278067961735" 17:48:06 DEBUG "value": [ 17:48:06 DEBUG ] 17:48:06 DEBUG 1713545286, 17:48:06 DEBUG } 17:48:06 DEBUG "74777965.26666667" 17:48:06 DEBUG ] 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG } 17:48:06 DEBUG } 17:48:06 DEBUG ] 17:48:06 DEBUG } 17:48:06 DEBUG } 17:48:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545287" 17:48:07 INFO [http_cmd]: http status code OK 17:48:07 DEBUG --- status code --- 17:48:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:07 DEBUG --- http response --- 17:48:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:48:07 DEBUG ----- output ----- 17:48:07 DEBUG { 17:48:07 DEBUG "status": "success", 17:48:07 DEBUG "data": { 17:48:07 DEBUG "resultType": "vector", 17:48:07 DEBUG "result": [ 17:48:07 DEBUG { 17:48:07 DEBUG "metric": { 17:48:07 DEBUG "instance": "10.142.0.107:9100" 17:48:07 DEBUG }, 17:48:07 DEBUG "value": [ 17:48:07 DEBUG 17:48:07 DEBUG 17:48:07 DEBUG 17:48:07 DEBUG [...] 17:48:07 DEBUG 17:48:07 DEBUG 17:48:07 DEBUG 17:48:07 DEBUG "instance": "10.142.0.43:9100" 17:48:07 DEBUG }, 17:48:07 DEBUG "value": [ 17:48:07 DEBUG 1713545287, 17:48:07 DEBUG "74771625.36666667" 17:48:07 DEBUG ] 17:48:07 DEBUG } 17:48:07 DEBUG ] 17:48:07 DEBUG } 17:48:07 DEBUG } 17:48:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545287" 17:48:07 INFO [http_cmd]: http status code OK 17:48:07 DEBUG --- status code --- 17:48:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:07 DEBUG --- http response --- 17:48:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:48:07 DEBUG ----- output ----- 17:48:07 DEBUG { 17:48:07 DEBUG "status": "success", 17:48:07 DEBUG "data": { 17:48:07 DEBUG "resultType": "vector", 17:48:07 DEBUG "result": [ 17:48:07 DEBUG { 17:48:07 DEBUG "metric": { 17:48:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:48:07 DEBUG }, 17:48:07 DEBUG "value": [ 17:48:07 DEBUG 17:48:07 DEBUG 17:48:07 DEBUG 17:48:07 DEBUG [...] 17:48:07 DEBUG 17:48:07 DEBUG 17:48:07 DEBUG 17:48:07 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:48:07 DEBUG }, 17:48:07 DEBUG "value": [ 17:48:07 DEBUG 1713545287, 17:48:07 DEBUG "0" 17:48:07 DEBUG ] 17:48:07 DEBUG } 17:48:07 DEBUG ] 17:48:07 DEBUG } 17:48:07 DEBUG } 17:48:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545287" 17:48:07 INFO [http_cmd]: http status code OK 17:48:07 DEBUG --- status code --- 17:48:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:07 DEBUG --- http response --- 17:48:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:48:07 DEBUG ----- output ----- 17:48:07 DEBUG { 17:48:07 DEBUG "status": "success", 17:48:07 DEBUG "data": { 17:48:07 DEBUG "resultType": "vector", 17:48:07 DEBUG "result": [ 17:48:07 DEBUG { 17:48:07 DEBUG "metric": { 17:48:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:48:07 DEBUG }, 17:48:07 DEBUG "value": [ 17:48:07 DEBUG 17:48:07 DEBUG 17:48:07 DEBUG 17:48:07 DEBUG [...] 17:48:07 DEBUG 17:48:07 DEBUG 17:48:07 DEBUG 17:48:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:48:07 DEBUG }, 17:48:07 DEBUG "value": [ 17:48:07 DEBUG 1713545287, 17:48:07 DEBUG "2499.1" 17:48:07 DEBUG ] 17:48:07 DEBUG } 17:48:07 DEBUG ] 17:48:07 DEBUG } 17:48:07 DEBUG } 17:48:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545287" 17:48:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545287" 17:48:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545287" 17:48:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545287" 17:48:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545287" 17:48:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545287" 17:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545288" 17:48:08 INFO [http_cmd]: http status code OK 17:48:08 INFO [http_cmd]: http status code OK 17:48:08 DEBUG --- status code --- 17:48:08 INFO [http_cmd]: http status code OK 17:48:08 DEBUG --- status code --- 17:48:08 INFO [http_cmd]: http status code OK 17:48:08 DEBUG --- status code --- 17:48:08 DEBUG --- status code --- 17:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:08 DEBUG --- http response --- 17:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:08 INFO [http_cmd]: http status code OK 17:48:08 INFO [http_cmd]: http status code OK 17:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:48:08 DEBUG --- http response --- 17:48:08 DEBUG --- status code --- 17:48:08 DEBUG --- status code --- 17:48:08 DEBUG --- http response --- 17:48:08 DEBUG --- http response --- 17:48:08 DEBUG ----- output ----- 17:48:08 INFO [http_cmd]: http status code OK 17:48:08 DEBUG --- status code --- 17:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:48:08 DEBUG { 17:48:08 DEBUG --- http response --- 17:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:08 DEBUG ----- output ----- 17:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:48:08 DEBUG ----- output ----- 17:48:08 DEBUG ----- output ----- 17:48:08 DEBUG { 17:48:08 DEBUG --- http response --- 17:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:48:08 DEBUG { 17:48:08 DEBUG "status": "success", 17:48:08 DEBUG "data": { 17:48:08 DEBUG "resultType": "vector", 17:48:08 DEBUG "result": [ 17:48:08 DEBUG ----- output ----- 17:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:48:08 DEBUG --- http response --- 17:48:08 DEBUG { 17:48:08 DEBUG "status": "success", 17:48:08 DEBUG { 17:48:08 DEBUG "status": "success", 17:48:08 DEBUG { 17:48:08 DEBUG ----- output ----- 17:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:48:08 DEBUG "status": "success", 17:48:08 DEBUG "data": { 17:48:08 DEBUG "metric": { 17:48:08 DEBUG "data": { 17:48:08 DEBUG "status": "success", 17:48:08 DEBUG { 17:48:08 DEBUG ----- output ----- 17:48:08 DEBUG "data": { 17:48:08 DEBUG "resultType": "vector", 17:48:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:48:08 DEBUG "resultType": "vector", 17:48:08 DEBUG "data": { 17:48:08 DEBUG "resultType": "vector", 17:48:08 DEBUG { 17:48:08 DEBUG "resultType": "vector", 17:48:08 DEBUG "result": [ 17:48:08 DEBUG }, 17:48:08 DEBUG "result": [ 17:48:08 DEBUG "result": [ 17:48:08 DEBUG "status": "success", 17:48:08 DEBUG "status": "success", 17:48:08 DEBUG "result": [ 17:48:08 DEBUG { 17:48:08 DEBUG "value": [ 17:48:08 DEBUG { 17:48:08 DEBUG { 17:48:08 DEBUG "data": { 17:48:08 DEBUG "data": { 17:48:08 DEBUG { 17:48:08 DEBUG "metric": { 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG "metric": { 17:48:08 DEBUG "resultType": "vector", 17:48:08 DEBUG "resultType": "vector", 17:48:08 DEBUG "metric": { 17:48:08 DEBUG "pod": "ds-cts-0" 17:48:08 DEBUG "metric": { 17:48:08 DEBUG 17:48:08 DEBUG "instance": "10.142.0.107:9100" 17:48:08 DEBUG "result": [ 17:48:08 DEBUG "result": [ 17:48:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:48:08 DEBUG }, 17:48:08 DEBUG "value": [ 17:48:08 DEBUG "pod": "ds-idrepo-0" 17:48:08 DEBUG }, 17:48:08 DEBUG { 17:48:08 DEBUG "metric": { 17:48:08 DEBUG }, 17:48:08 DEBUG "value": [ 17:48:08 DEBUG 17:48:08 DEBUG }, 17:48:08 DEBUG "value": [ 17:48:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:48:08 DEBUG }, 17:48:08 DEBUG 17:48:08 DEBUG [...] 17:48:08 DEBUG 17:48:08 DEBUG "value": [ 17:48:08 DEBUG 17:48:08 DEBUG { 17:48:08 DEBUG "value": [ 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG "metric": { 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG [...] 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:48:08 DEBUG }, 17:48:08 DEBUG [...] 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG "value": [ 17:48:08 DEBUG 17:48:08 DEBUG "pod": "ds-cts-2" 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG }, 17:48:08 DEBUG [...] 17:48:08 DEBUG [...] 17:48:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:48:08 DEBUG [...] 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG "value": [ 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG }, 17:48:08 DEBUG 17:48:08 DEBUG [...] 17:48:08 DEBUG 17:48:08 DEBUG 1713545287, 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG "value": [ 17:48:08 DEBUG 1713545287, 17:48:08 DEBUG 17:48:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:48:08 DEBUG "226.7" 17:48:08 DEBUG ] 17:48:08 DEBUG } 17:48:08 DEBUG ] 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG }, 17:48:08 DEBUG 17:48:08 DEBUG "2508.366666666667" 17:48:08 DEBUG } 17:48:08 DEBUG 17:48:08 DEBUG 17:48:08 DEBUG "instance": "10.142.0.43:9100" 17:48:08 DEBUG "value": [ 17:48:08 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:48:08 DEBUG }, 17:48:08 DEBUG } 17:48:08 DEBUG "pod": "ds-idrepo-2" 17:48:08 DEBUG 17:48:08 DEBUG }, 17:48:08 DEBUG 1713545287, 17:48:08 DEBUG ] 17:48:08 DEBUG "value": [ 17:48:08 DEBUG 1713545287, 17:48:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:48:08 DEBUG "value": [ 17:48:08 DEBUG "0.023" 17:48:08 DEBUG } 17:48:08 DEBUG }, 17:48:08 DEBUG "0.0020642925131554237" 17:48:08 DEBUG }, 17:48:08 DEBUG 1713545287, 17:48:08 DEBUG ] 17:48:08 DEBUG ] 17:48:08 DEBUG } 17:48:08 DEBUG ] 17:48:08 DEBUG "value": [ 17:48:08 DEBUG "0.9386666666666741" 17:48:08 DEBUG } 17:48:08 DEBUG } 17:48:08 DEBUG "value": [ 17:48:08 DEBUG } 17:48:08 DEBUG 1713545288, 17:48:08 DEBUG ] 17:48:08 DEBUG ] 17:48:08 DEBUG 1713545287, 17:48:08 DEBUG ] 17:48:08 DEBUG "8.029059859158027" 17:48:08 DEBUG } 17:48:08 DEBUG } 17:48:08 DEBUG "0" 17:48:08 DEBUG } 17:48:08 DEBUG ] 17:48:08 DEBUG ] 17:48:08 DEBUG } 17:48:08 DEBUG ] 17:48:08 DEBUG } 17:48:08 DEBUG } 17:48:08 DEBUG } 17:48:08 DEBUG ] 17:48:08 DEBUG } 17:48:08 DEBUG ] 17:48:08 DEBUG } 17:48:08 DEBUG } 17:48:08 DEBUG } 17:48:08 DEBUG } 17:48:08 DEBUG } 17:48:08 INFO 17:48:08 INFO [loop_until]: kubectl --namespace=xlou top pods 17:48:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:48:08 INFO [loop_until]: OK (rc = 0) 17:48:08 DEBUG --- stdout --- 17:48:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 17660m 9020Mi am-6fd554f8f7-btd4s 17308m 9068Mi am-6fd554f8f7-wxcsj 19158m 9059Mi ds-cts-0 21100m 13397Mi ds-cts-1 21139m 13615Mi ds-cts-2 21713m 13304Mi ds-idrepo-0 6657m 3423Mi ds-idrepo-1 6734m 3395Mi ds-idrepo-2 7021m 3454Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8043m 2397Mi 17:48:08 DEBUG --- stderr --- 17:48:08 DEBUG 17:48:08 INFO 17:48:08 INFO [loop_until]: kubectl --namespace=xlou top node 17:48:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:48:08 INFO [loop_until]: OK (rc = 0) 17:48:08 DEBUG --- stdout --- 17:48:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6173Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7388m 24% 6993Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 47m 0% 2681Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 17539m 58% 12324Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 20836m 69% 16379Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 5238Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17508m 58% 11970Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 17001m 56% 3132Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19583m 65% 11294Mi 10% gke-xlou-cdm-ds-0f601751-j488 7619m 25% 6692Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21060m 70% 18373Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 7436m 24% 7239Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21381m 71% 15806Mi 14% gke-xlou-cdm-frontend-a8771548-i82n 8186m 51% 6121Mi 10% 17:48:08 DEBUG --- stderr --- 17:48:08 DEBUG 17:49:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545345" 17:49:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545345" 17:49:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545345" 17:49:05 INFO [http_cmd]: http status code OK 17:49:05 DEBUG --- status code --- 17:49:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:05 DEBUG --- http response --- 17:49:05 INFO [http_cmd]: http status code OK 17:49:05 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:49:05 DEBUG --- status code --- 17:49:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:05 DEBUG --- http response --- 17:49:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:49:05 DEBUG ----- output ----- 17:49:05 DEBUG { 17:49:05 ERROR [http_cmd]: ERROR 17:49:05 DEBUG "status": "success", 17:49:05 DEBUG "data": { 17:49:05 ERROR --- status code --- 17:49:05 DEBUG "resultType": "vector", 17:49:05 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:49:05 ERROR --- http response --- 17:49:05 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:49:05 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:49:05 DEBUG "result": [ 17:49:05 DEBUG { 17:49:05 DEBUG "metric": { 17:49:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:49:05 DEBUG }, 17:49:05 DEBUG "value": [ 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG [...] 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:49:05 DEBUG }, 17:49:05 DEBUG "value": [ 17:49:05 DEBUG 1713545345, 17:49:05 DEBUG "2358.1" 17:49:05 DEBUG ] 17:49:05 DEBUG } 17:49:05 DEBUG ] 17:49:05 DEBUG } 17:49:05 DEBUG } 17:49:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545345" 17:49:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545345" 17:49:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545345" 17:49:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545345" 17:49:05 INFO [http_cmd]: http status code OK 17:49:05 DEBUG --- status code --- 17:49:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:05 DEBUG --- http response --- 17:49:05 INFO [http_cmd]: http status code OK 17:49:05 DEBUG --- status code --- 17:49:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:05 DEBUG --- http response --- 17:49:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:49:05 INFO [http_cmd]: http status code OK 17:49:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:49:05 INFO [http_cmd]: http status code OK 17:49:05 DEBUG ----- output ----- 17:49:05 DEBUG --- status code --- 17:49:05 DEBUG ----- output ----- 17:49:05 DEBUG --- status code --- 17:49:05 DEBUG { 17:49:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:05 DEBUG { 17:49:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:05 DEBUG "status": "success", 17:49:05 DEBUG --- http response --- 17:49:05 DEBUG "status": "success", 17:49:05 DEBUG --- http response --- 17:49:05 DEBUG "data": { 17:49:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:49:05 DEBUG "data": { 17:49:05 DEBUG "resultType": "vector", 17:49:05 DEBUG "resultType": "vector", 17:49:05 DEBUG ----- output ----- 17:49:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:49:05 DEBUG "result": [ 17:49:05 DEBUG "result": [ 17:49:05 DEBUG { 17:49:05 DEBUG ----- output ----- 17:49:05 DEBUG { 17:49:05 DEBUG { 17:49:05 DEBUG "status": "success", 17:49:05 DEBUG { 17:49:05 DEBUG "metric": { 17:49:05 DEBUG "metric": { 17:49:05 DEBUG "data": { 17:49:05 DEBUG "resultType": "vector", 17:49:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:49:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:49:05 DEBUG "result": [ 17:49:05 DEBUG }, 17:49:05 DEBUG "value": [ 17:49:05 DEBUG "status": "success", 17:49:05 DEBUG { 17:49:05 DEBUG }, 17:49:05 DEBUG 17:49:05 DEBUG "data": { 17:49:05 DEBUG "metric": { 17:49:05 DEBUG "value": [ 17:49:05 DEBUG 17:49:05 DEBUG "resultType": "vector", 17:49:05 DEBUG "result": [ 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG { 17:49:05 DEBUG "pod": "ds-idrepo-0" 17:49:05 DEBUG 17:49:05 DEBUG [...] 17:49:05 DEBUG "metric": { 17:49:05 DEBUG }, 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG "pod": "ds-idrepo-0" 17:49:05 DEBUG "value": [ 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG }, 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG "value": [ 17:49:05 DEBUG 17:49:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:49:05 DEBUG [...] 17:49:05 DEBUG 17:49:05 DEBUG [...] 17:49:05 DEBUG }, 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG [...] 17:49:05 DEBUG 17:49:05 DEBUG "value": [ 17:49:05 DEBUG 17:49:05 DEBUG "pod": "idm-b85f9d978-txpwv" 17:49:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:49:05 DEBUG 1713545345, 17:49:05 DEBUG }, 17:49:05 DEBUG 17:49:05 DEBUG }, 17:49:05 DEBUG "0" 17:49:05 DEBUG "value": [ 17:49:05 DEBUG 17:49:05 DEBUG "value": [ 17:49:05 DEBUG ] 17:49:05 DEBUG 1713545345, 17:49:05 DEBUG 17:49:05 DEBUG 1713545345, 17:49:05 DEBUG } 17:49:05 DEBUG "0" 17:49:05 DEBUG ] 17:49:05 DEBUG } 17:49:05 DEBUG ] 17:49:05 DEBUG ] 17:49:05 DEBUG "2510.866666666667" 17:49:05 DEBUG } 17:49:05 DEBUG "pod": "ds-cts-2" 17:49:05 DEBUG } 17:49:05 DEBUG ] 17:49:05 DEBUG } 17:49:05 DEBUG }, 17:49:05 DEBUG } 17:49:05 DEBUG } 17:49:05 DEBUG "value": [ 17:49:05 DEBUG ] 17:49:05 DEBUG 1713545345, 17:49:05 DEBUG } 17:49:05 DEBUG "0" 17:49:05 DEBUG } 17:49:05 DEBUG ] 17:49:05 DEBUG } 17:49:05 DEBUG ] 17:49:05 DEBUG } 17:49:05 DEBUG } 17:49:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545345" 17:49:05 INFO [http_cmd]: http status code OK 17:49:05 DEBUG --- status code --- 17:49:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:05 DEBUG --- http response --- 17:49:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:49:05 DEBUG ----- output ----- 17:49:05 DEBUG { 17:49:05 DEBUG "status": "success", 17:49:05 DEBUG "data": { 17:49:05 DEBUG "resultType": "vector", 17:49:05 DEBUG "result": [ 17:49:05 DEBUG { 17:49:05 DEBUG "metric": { 17:49:05 DEBUG "instance": "10.142.0.107:9100" 17:49:05 DEBUG }, 17:49:05 DEBUG "value": [ 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG [...] 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG "instance": "10.142.0.43:9100" 17:49:05 DEBUG }, 17:49:05 DEBUG "value": [ 17:49:05 DEBUG 1713545345, 17:49:05 DEBUG "0.710000000000008" 17:49:05 DEBUG ] 17:49:05 DEBUG } 17:49:05 DEBUG ] 17:49:05 DEBUG } 17:49:05 DEBUG } 17:49:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545345" 17:49:05 INFO [http_cmd]: http status code OK 17:49:05 DEBUG --- status code --- 17:49:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:05 DEBUG --- http response --- 17:49:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:49:05 DEBUG ----- output ----- 17:49:05 DEBUG { 17:49:05 DEBUG "status": "success", 17:49:05 DEBUG "data": { 17:49:05 DEBUG "resultType": "vector", 17:49:05 DEBUG "result": [ 17:49:05 DEBUG { 17:49:05 DEBUG "metric": { 17:49:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:49:05 DEBUG }, 17:49:05 DEBUG "value": [ 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG [...] 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG 17:49:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:49:05 DEBUG }, 17:49:05 DEBUG "value": [ 17:49:05 DEBUG 1713545345, 17:49:05 DEBUG "4235299" 17:49:05 DEBUG ] 17:49:05 DEBUG } 17:49:05 DEBUG ] 17:49:05 DEBUG } 17:49:05 DEBUG } 17:49:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545346" 17:49:06 INFO [http_cmd]: http status code OK 17:49:06 DEBUG --- status code --- 17:49:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:06 DEBUG --- http response --- 17:49:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:49:06 DEBUG ----- output ----- 17:49:06 DEBUG { 17:49:06 DEBUG "status": "success", 17:49:06 DEBUG "data": { 17:49:06 DEBUG "resultType": "vector", 17:49:06 DEBUG "result": [ 17:49:06 DEBUG { 17:49:06 DEBUG "metric": { 17:49:06 DEBUG "instance": "10.142.0.107:9100" 17:49:06 DEBUG }, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG [...] 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG "instance": "10.142.0.43:9100" 17:49:06 DEBUG }, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 1713545346, 17:49:06 DEBUG "1779132642304" 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG } 17:49:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545346" 17:49:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545346" 17:49:06 INFO [http_cmd]: http status code OK 17:49:06 DEBUG --- status code --- 17:49:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:06 DEBUG --- http response --- 17:49:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:49:06 DEBUG ----- output ----- 17:49:06 DEBUG { 17:49:06 DEBUG "status": "success", 17:49:06 DEBUG "data": { 17:49:06 DEBUG "resultType": "vector", 17:49:06 DEBUG "result": [ 17:49:06 DEBUG { 17:49:06 DEBUG "metric": { 17:49:06 DEBUG "pod": "ds-idrepo-0" 17:49:06 DEBUG }, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG [...] 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG "pod": "ds-idrepo-2" 17:49:06 DEBUG }, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 1713545346, 17:49:06 DEBUG "0" 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG } 17:49:06 INFO [http_cmd]: http status code OK 17:49:06 DEBUG --- status code --- 17:49:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:06 DEBUG --- http response --- 17:49:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:49:06 DEBUG ----- output ----- 17:49:06 DEBUG { 17:49:06 DEBUG "status": "success", 17:49:06 DEBUG "data": { 17:49:06 DEBUG "resultType": "vector", 17:49:06 DEBUG "result": [ 17:49:06 DEBUG { 17:49:06 DEBUG "metric": { 17:49:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:49:06 DEBUG }, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG [...] 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:49:06 DEBUG }, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 1713545346, 17:49:06 DEBUG "4100796416" 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG } 17:49:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545346" 17:49:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545346" 17:49:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545346" 17:49:06 INFO [http_cmd]: http status code OK 17:49:06 DEBUG --- status code --- 17:49:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:06 DEBUG --- http response --- 17:49:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:49:06 DEBUG ----- output ----- 17:49:06 DEBUG { 17:49:06 DEBUG "status": "success", 17:49:06 DEBUG "data": { 17:49:06 DEBUG "resultType": "vector", 17:49:06 DEBUG "result": [ 17:49:06 DEBUG { 17:49:06 DEBUG "metric": { 17:49:06 DEBUG "pod": "ds-cts-0" 17:49:06 DEBUG }, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG [...] 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG "pod": "ds-cts-2" 17:49:06 DEBUG }, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 1713545346, 17:49:06 DEBUG "0.2086000000000003" 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG } 17:49:06 INFO [http_cmd]: http status code OK 17:49:06 DEBUG --- status code --- 17:49:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:06 DEBUG --- http response --- 17:49:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:49:06 DEBUG ----- output ----- 17:49:06 DEBUG { 17:49:06 DEBUG "status": "success", 17:49:06 DEBUG "data": { 17:49:06 DEBUG "resultType": "vector", 17:49:06 DEBUG "result": [ 17:49:06 DEBUG { 17:49:06 DEBUG "metric": { 17:49:06 DEBUG "pod": "ds-cts-0" 17:49:06 DEBUG }, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG [...] 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG "pod": "ds-cts-2" 17:49:06 DEBUG }, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 1713545346, 17:49:06 DEBUG "31.9" 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG } 17:49:06 INFO [http_cmd]: http status code OK 17:49:06 DEBUG --- status code --- 17:49:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:06 DEBUG --- http response --- 17:49:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:49:06 DEBUG ----- output ----- 17:49:06 DEBUG { 17:49:06 DEBUG "status": "success", 17:49:06 DEBUG "data": { 17:49:06 DEBUG "resultType": "vector", 17:49:06 DEBUG "result": [ 17:49:06 DEBUG { 17:49:06 DEBUG "metric": { 17:49:06 DEBUG "pod": "ds-idrepo-0" 17:49:06 DEBUG }, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG [...] 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG "pod": "ds-idrepo-2" 17:49:06 DEBUG }, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 1713545346, 17:49:06 DEBUG "0" 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG } 17:49:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545346" 17:49:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545346" 17:49:06 INFO [http_cmd]: http status code OK 17:49:06 DEBUG --- status code --- 17:49:06 INFO [http_cmd]: http status code OK 17:49:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:06 DEBUG --- status code --- 17:49:06 DEBUG --- http response --- 17:49:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:49:06 DEBUG --- http response --- 17:49:06 DEBUG ----- output ----- 17:49:06 DEBUG { 17:49:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:49:06 DEBUG "status": "success", 17:49:06 DEBUG ----- output ----- 17:49:06 DEBUG "data": { 17:49:06 DEBUG { 17:49:06 DEBUG "resultType": "vector", 17:49:06 DEBUG "status": "success", 17:49:06 DEBUG "result": [ 17:49:06 DEBUG "data": { 17:49:06 DEBUG { 17:49:06 DEBUG "resultType": "vector", 17:49:06 DEBUG "metric": { 17:49:06 DEBUG "result": [ 17:49:06 DEBUG "instance": "10.142.0.107:9100" 17:49:06 DEBUG { 17:49:06 DEBUG }, 17:49:06 DEBUG "metric": { 17:49:06 DEBUG "value": [ 17:49:06 DEBUG "pod": "ds-idrepo-0" 17:49:06 DEBUG }, 17:49:06 DEBUG 17:49:06 DEBUG "value": [ 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG [...] 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG [...] 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG 17:49:06 DEBUG "instance": "10.142.0.43:9100" 17:49:06 DEBUG 17:49:06 DEBUG }, 17:49:06 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:49:06 DEBUG "value": [ 17:49:06 DEBUG }, 17:49:06 DEBUG 1713545346, 17:49:06 DEBUG "value": [ 17:49:06 DEBUG "74325445.5" 17:49:06 DEBUG 1713545346, 17:49:06 DEBUG ] 17:49:06 DEBUG "1443.8804286520021" 17:49:06 DEBUG } 17:49:06 DEBUG ] 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG } 17:49:06 DEBUG ] 17:49:06 DEBUG } 17:49:06 DEBUG } 17:49:06 DEBUG } 17:49:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545347" 17:49:07 INFO [http_cmd]: http status code OK 17:49:07 DEBUG --- status code --- 17:49:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:07 DEBUG --- http response --- 17:49:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:49:07 DEBUG ----- output ----- 17:49:07 DEBUG { 17:49:07 DEBUG "status": "success", 17:49:07 DEBUG "data": { 17:49:07 DEBUG "resultType": "vector", 17:49:07 DEBUG "result": [ 17:49:07 DEBUG { 17:49:07 DEBUG "metric": { 17:49:07 DEBUG "instance": "10.142.0.107:9100" 17:49:07 DEBUG }, 17:49:07 DEBUG "value": [ 17:49:07 DEBUG 17:49:07 DEBUG 17:49:07 DEBUG 17:49:07 DEBUG [...] 17:49:07 DEBUG 17:49:07 DEBUG 17:49:07 DEBUG 17:49:07 DEBUG "instance": "10.142.0.43:9100" 17:49:07 DEBUG }, 17:49:07 DEBUG "value": [ 17:49:07 DEBUG 1713545347, 17:49:07 DEBUG "74318563.86666666" 17:49:07 DEBUG ] 17:49:07 DEBUG } 17:49:07 DEBUG ] 17:49:07 DEBUG } 17:49:07 DEBUG } 17:49:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545347" 17:49:07 INFO [http_cmd]: http status code OK 17:49:07 DEBUG --- status code --- 17:49:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:07 DEBUG --- http response --- 17:49:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:49:07 DEBUG ----- output ----- 17:49:07 DEBUG { 17:49:07 DEBUG "status": "success", 17:49:07 DEBUG "data": { 17:49:07 DEBUG "resultType": "vector", 17:49:07 DEBUG "result": [ 17:49:07 DEBUG { 17:49:07 DEBUG "metric": { 17:49:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:49:07 DEBUG }, 17:49:07 DEBUG "value": [ 17:49:07 DEBUG 17:49:07 DEBUG 17:49:07 DEBUG 17:49:07 DEBUG [...] 17:49:07 DEBUG 17:49:07 DEBUG 17:49:07 DEBUG 17:49:07 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:49:07 DEBUG }, 17:49:07 DEBUG "value": [ 17:49:07 DEBUG 1713545347, 17:49:07 DEBUG "0" 17:49:07 DEBUG ] 17:49:07 DEBUG } 17:49:07 DEBUG ] 17:49:07 DEBUG } 17:49:07 DEBUG } 17:49:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545347" 17:49:07 INFO [http_cmd]: http status code OK 17:49:07 DEBUG --- status code --- 17:49:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:07 DEBUG --- http response --- 17:49:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:49:07 DEBUG ----- output ----- 17:49:07 DEBUG { 17:49:07 DEBUG "status": "success", 17:49:07 DEBUG "data": { 17:49:07 DEBUG "resultType": "vector", 17:49:07 DEBUG "result": [ 17:49:07 DEBUG { 17:49:07 DEBUG "metric": { 17:49:07 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:49:07 DEBUG }, 17:49:07 DEBUG "value": [ 17:49:07 DEBUG 17:49:07 DEBUG 17:49:07 DEBUG 17:49:07 DEBUG [...] 17:49:07 DEBUG 17:49:07 DEBUG 17:49:07 DEBUG 17:49:07 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:49:07 DEBUG }, 17:49:07 DEBUG "value": [ 17:49:07 DEBUG 1713545347, 17:49:07 DEBUG "2510.633333333333" 17:49:07 DEBUG ] 17:49:07 DEBUG } 17:49:07 DEBUG ] 17:49:07 DEBUG } 17:49:07 DEBUG } 17:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545348" 17:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545348" 17:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545348" 17:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545348" 17:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545348" 17:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545348" 17:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545348" 17:49:08 INFO [http_cmd]: http status code OK 17:49:08 DEBUG --- status code --- 17:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:08 INFO [http_cmd]: http status code OK 17:49:08 DEBUG --- status code --- 17:49:08 INFO [http_cmd]: http status code OK 17:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:08 DEBUG --- status code --- 17:49:08 INFO [http_cmd]: http status code OK 17:49:08 DEBUG --- http response --- 17:49:08 DEBUG --- http response --- 17:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:08 DEBUG --- status code --- 17:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:49:08 DEBUG --- http response --- 17:49:08 INFO [http_cmd]: http status code OK 17:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:08 INFO [http_cmd]: http status code OK 17:49:08 DEBUG ----- output ----- 17:49:08 DEBUG { 17:49:08 DEBUG --- status code --- 17:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:49:08 DEBUG --- http response --- 17:49:08 INFO [http_cmd]: http status code OK 17:49:08 DEBUG --- status code --- 17:49:08 DEBUG "status": "success", 17:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:08 DEBUG ----- output ----- 17:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:49:08 DEBUG --- status code --- 17:49:08 DEBUG ----- output ----- 17:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:08 DEBUG "data": { 17:49:08 DEBUG --- http response --- 17:49:08 DEBUG { 17:49:08 DEBUG ----- output ----- 17:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:49:08 DEBUG { 17:49:08 DEBUG --- http response --- 17:49:08 DEBUG "resultType": "vector", 17:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:49:08 DEBUG "status": "success", 17:49:08 DEBUG { 17:49:08 DEBUG --- http response --- 17:49:08 DEBUG "status": "success", 17:49:08 DEBUG "result": [ 17:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:49:08 DEBUG ----- output ----- 17:49:08 DEBUG "data": { 17:49:08 DEBUG "status": "success", 17:49:08 DEBUG "data": { 17:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:49:08 DEBUG { 17:49:08 DEBUG ----- output ----- 17:49:08 DEBUG { 17:49:08 DEBUG "resultType": "vector", 17:49:08 DEBUG "data": { 17:49:08 DEBUG "resultType": "vector", 17:49:08 DEBUG ----- output ----- 17:49:08 DEBUG "metric": { 17:49:08 DEBUG { 17:49:08 DEBUG "status": "success", 17:49:08 DEBUG "result": [ 17:49:08 DEBUG "resultType": "vector", 17:49:08 DEBUG "result": [ 17:49:08 DEBUG { 17:49:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:49:08 DEBUG "status": "success", 17:49:08 DEBUG "data": { 17:49:08 DEBUG { 17:49:08 DEBUG "result": [ 17:49:08 DEBUG { 17:49:08 DEBUG "status": "success", 17:49:08 DEBUG }, 17:49:08 DEBUG "data": { 17:49:08 DEBUG "resultType": "vector", 17:49:08 DEBUG "metric": { 17:49:08 DEBUG { 17:49:08 DEBUG "metric": { 17:49:08 DEBUG "data": { 17:49:08 DEBUG "value": [ 17:49:08 DEBUG "resultType": "vector", 17:49:08 DEBUG "result": [ 17:49:08 DEBUG { 17:49:08 DEBUG "instance": "10.142.0.107:9100" 17:49:08 DEBUG "pod": "ds-cts-0" 17:49:08 DEBUG }, 17:49:08 DEBUG 17:49:08 DEBUG "result": [ 17:49:08 DEBUG "metric": { 17:49:08 DEBUG "metric": { 17:49:08 DEBUG }, 17:49:08 DEBUG "value": [ 17:49:08 DEBUG "resultType": "vector", 17:49:08 DEBUG 17:49:08 DEBUG { 17:49:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:49:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:49:08 DEBUG "value": [ 17:49:08 DEBUG 17:49:08 DEBUG "result": [ 17:49:08 DEBUG { 17:49:08 DEBUG "metric": { 17:49:08 DEBUG }, 17:49:08 DEBUG }, 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG "metric": { 17:49:08 DEBUG "pod": "ds-idrepo-0" 17:49:08 DEBUG 17:49:08 DEBUG "value": [ 17:49:08 DEBUG "value": [ 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:49:08 DEBUG }, 17:49:08 DEBUG [...] 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG [...] 17:49:08 DEBUG }, 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG [...] 17:49:08 DEBUG "value": [ 17:49:08 DEBUG 17:49:08 DEBUG "value": [ 17:49:08 DEBUG 17:49:08 DEBUG [...] 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG [...] 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG }, 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG [...] 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG [...] 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG 17:49:08 DEBUG "pod": "ds-cts-2" 17:49:08 DEBUG 17:49:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:49:08 DEBUG "value": [ 17:49:08 DEBUG 1713545348, 17:49:08 DEBUG "instance": "10.142.0.43:9100" 17:49:08 DEBUG 17:49:08 DEBUG }, 17:49:08 DEBUG 17:49:08 DEBUG }, 17:49:08 DEBUG "value": [ 17:49:08 DEBUG }, 17:49:08 DEBUG 17:49:08 DEBUG "value": [ 17:49:08 DEBUG 17:49:08 DEBUG "0.024" 17:49:08 DEBUG ] 17:49:08 DEBUG 1713545348, 17:49:08 DEBUG "pod": "ds-idrepo-2" 17:49:08 DEBUG 17:49:08 DEBUG 1713545348, 17:49:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:49:08 DEBUG "value": [ 17:49:08 DEBUG } 17:49:08 DEBUG "2511.0333333333333" 17:49:08 DEBUG }, 17:49:08 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:49:08 DEBUG "233.13333333333333" 17:49:08 DEBUG }, 17:49:08 DEBUG 1713545348, 17:49:08 DEBUG ] 17:49:08 DEBUG ] 17:49:08 DEBUG "value": [ 17:49:08 DEBUG }, 17:49:08 DEBUG ] 17:49:08 DEBUG "value": [ 17:49:08 DEBUG "0.938633333333334" 17:49:08 DEBUG } 17:49:08 DEBUG } 17:49:08 DEBUG 1713545348, 17:49:08 DEBUG "value": [ 17:49:08 DEBUG } 17:49:08 DEBUG 1713545348, 17:49:08 DEBUG ] 17:49:08 DEBUG } 17:49:08 DEBUG ] 17:49:08 DEBUG "0" 17:49:08 DEBUG 1713545348, 17:49:08 DEBUG ] 17:49:08 DEBUG "8.028890891099588" 17:49:08 DEBUG } 17:49:08 DEBUG } 17:49:08 DEBUG ] 17:49:08 DEBUG "0" 17:49:08 DEBUG } 17:49:08 DEBUG ] 17:49:08 DEBUG ] 17:49:08 DEBUG } 17:49:08 DEBUG } 17:49:08 DEBUG ] 17:49:08 DEBUG } 17:49:08 DEBUG } 17:49:08 DEBUG } 17:49:08 DEBUG ] 17:49:08 DEBUG } 17:49:08 DEBUG ] 17:49:08 DEBUG } 17:49:08 DEBUG } 17:49:08 DEBUG ] 17:49:08 DEBUG } 17:49:08 DEBUG } 17:49:08 DEBUG } 17:49:08 DEBUG } 17:49:08 DEBUG } 17:49:08 INFO 17:49:08 INFO [loop_until]: kubectl --namespace=xlou top pods 17:49:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:49:08 INFO [loop_until]: OK (rc = 0) 17:49:08 DEBUG --- stdout --- 17:49:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 14548m 9025Mi am-6fd554f8f7-btd4s 15247m 9071Mi am-6fd554f8f7-wxcsj 18094m 9065Mi ds-cts-0 20509m 13481Mi ds-cts-1 21022m 13587Mi ds-cts-2 21175m 13192Mi ds-idrepo-0 6449m 3424Mi ds-idrepo-1 5926m 3394Mi ds-idrepo-2 6223m 3454Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8062m 3373Mi 17:49:08 DEBUG --- stderr --- 17:49:08 DEBUG 17:49:09 INFO 17:49:09 INFO [loop_until]: kubectl --namespace=xlou top node 17:49:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:49:09 INFO [loop_until]: OK (rc = 0) 17:49:09 DEBUG --- stdout --- 17:49:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6188Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6078m 20% 6990Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 47m 0% 2694Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 15353m 51% 12350Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 20656m 69% 16350Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 139m 0% 5241Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 15532m 52% 11994Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14736m 49% 3679Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 17089m 57% 11306Mi 10% gke-xlou-cdm-ds-0f601751-j488 7225m 24% 6691Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 21165m 70% 18378Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 6370m 21% 7238Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21284m 71% 15588Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 8157m 51% 7012Mi 11% 17:49:09 DEBUG --- stderr --- 17:49:09 DEBUG 17:50:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545405" 17:50:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545405" 17:50:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545405" 17:50:05 ERROR [http_cmd]: ERROR 17:50:05 INFO [http_cmd]: http status code OK 17:50:05 INFO [http_cmd]: http status code OK 17:50:05 ERROR --- status code --- 17:50:05 DEBUG --- status code --- 17:50:05 DEBUG --- status code --- 17:50:05 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:50:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:05 ERROR --- http response --- 17:50:05 DEBUG --- http response --- 17:50:05 DEBUG --- http response --- 17:50:05 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:50:05 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:50:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:50:05 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:50:05 DEBUG ----- output ----- 17:50:05 DEBUG { 17:50:05 DEBUG "status": "success", 17:50:05 DEBUG "data": { 17:50:05 DEBUG "resultType": "vector", 17:50:05 DEBUG "result": [ 17:50:05 DEBUG { 17:50:05 DEBUG "metric": { 17:50:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:50:05 DEBUG }, 17:50:05 DEBUG "value": [ 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG [...] 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:50:05 DEBUG }, 17:50:05 DEBUG "value": [ 17:50:05 DEBUG 1713545405, 17:50:05 DEBUG "2339.6666666666665" 17:50:05 DEBUG ] 17:50:05 DEBUG } 17:50:05 DEBUG ] 17:50:05 DEBUG } 17:50:05 DEBUG } 17:50:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545405" 17:50:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545405" 17:50:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545405" 17:50:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545405" 17:50:05 INFO [http_cmd]: http status code OK 17:50:05 INFO [http_cmd]: http status code OK 17:50:05 INFO [http_cmd]: http status code OK 17:50:05 INFO [http_cmd]: http status code OK 17:50:05 DEBUG --- status code --- 17:50:05 DEBUG --- status code --- 17:50:05 DEBUG --- status code --- 17:50:05 DEBUG --- status code --- 17:50:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:05 DEBUG --- http response --- 17:50:05 DEBUG --- http response --- 17:50:05 DEBUG --- http response --- 17:50:05 DEBUG --- http response --- 17:50:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:50:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:50:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:50:05 DEBUG ----- output ----- 17:50:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:50:05 DEBUG ----- output ----- 17:50:05 DEBUG ----- output ----- 17:50:05 DEBUG { 17:50:05 DEBUG ----- output ----- 17:50:05 DEBUG { 17:50:05 DEBUG { 17:50:05 DEBUG "status": "success", 17:50:05 DEBUG { 17:50:05 DEBUG "status": "success", 17:50:05 DEBUG "status": "success", 17:50:05 DEBUG "data": { 17:50:05 DEBUG "status": "success", 17:50:05 DEBUG "data": { 17:50:05 DEBUG "data": { 17:50:05 DEBUG "resultType": "vector", 17:50:05 DEBUG "data": { 17:50:05 DEBUG "resultType": "vector", 17:50:05 DEBUG "resultType": "vector", 17:50:05 DEBUG "result": [ 17:50:05 DEBUG "resultType": "vector", 17:50:05 DEBUG "result": [ 17:50:05 DEBUG "result": [ 17:50:05 DEBUG { 17:50:05 DEBUG "result": [ 17:50:05 DEBUG { 17:50:05 DEBUG { 17:50:05 DEBUG "metric": { 17:50:05 DEBUG { 17:50:05 DEBUG "metric": { 17:50:05 DEBUG "metric": { 17:50:05 DEBUG "pod": "ds-idrepo-0" 17:50:05 DEBUG "metric": { 17:50:05 DEBUG "pod": "ds-idrepo-0" 17:50:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:50:05 DEBUG }, 17:50:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:50:05 DEBUG }, 17:50:05 DEBUG }, 17:50:05 DEBUG "value": [ 17:50:05 DEBUG }, 17:50:05 DEBUG "value": [ 17:50:05 DEBUG "value": [ 17:50:05 DEBUG 17:50:05 DEBUG "value": [ 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG [...] 17:50:05 DEBUG 17:50:05 DEBUG [...] 17:50:05 DEBUG [...] 17:50:05 DEBUG 17:50:05 DEBUG [...] 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG "pod": "ds-cts-2" 17:50:05 DEBUG 17:50:05 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 17:50:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:50:05 DEBUG }, 17:50:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:50:05 DEBUG }, 17:50:05 DEBUG }, 17:50:05 DEBUG "value": [ 17:50:05 DEBUG }, 17:50:05 DEBUG "value": [ 17:50:05 DEBUG "value": [ 17:50:05 DEBUG 1713545405, 17:50:05 DEBUG "value": [ 17:50:05 DEBUG 1713545405, 17:50:05 DEBUG 1713545405, 17:50:05 DEBUG "0" 17:50:05 DEBUG 1713545405, 17:50:05 DEBUG "106.93966270404782" 17:50:05 DEBUG "0" 17:50:05 DEBUG ] 17:50:05 DEBUG "1367.7333333333333" 17:50:05 DEBUG ] 17:50:05 DEBUG ] 17:50:05 DEBUG } 17:50:05 DEBUG ] 17:50:05 DEBUG } 17:50:05 DEBUG } 17:50:05 DEBUG ] 17:50:05 DEBUG } 17:50:05 DEBUG ] 17:50:05 DEBUG ] 17:50:05 DEBUG } 17:50:05 DEBUG ] 17:50:05 DEBUG } 17:50:05 DEBUG } 17:50:05 DEBUG } 17:50:05 DEBUG } 17:50:05 DEBUG } 17:50:05 DEBUG } 17:50:05 DEBUG } 17:50:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545405" 17:50:05 INFO [http_cmd]: http status code OK 17:50:05 DEBUG --- status code --- 17:50:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:05 DEBUG --- http response --- 17:50:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:50:05 DEBUG ----- output ----- 17:50:05 DEBUG { 17:50:05 DEBUG "status": "success", 17:50:05 DEBUG "data": { 17:50:05 DEBUG "resultType": "vector", 17:50:05 DEBUG "result": [ 17:50:05 DEBUG { 17:50:05 DEBUG "metric": { 17:50:05 DEBUG "instance": "10.142.0.107:9100" 17:50:05 DEBUG }, 17:50:05 DEBUG "value": [ 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG [...] 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG "instance": "10.142.0.43:9100" 17:50:05 DEBUG }, 17:50:05 DEBUG "value": [ 17:50:05 DEBUG 1713545405, 17:50:05 DEBUG "0.8026666666666663" 17:50:05 DEBUG ] 17:50:05 DEBUG } 17:50:05 DEBUG ] 17:50:05 DEBUG } 17:50:05 DEBUG } 17:50:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545405" 17:50:05 INFO [http_cmd]: http status code OK 17:50:05 DEBUG --- status code --- 17:50:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:05 DEBUG --- http response --- 17:50:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:50:05 DEBUG ----- output ----- 17:50:05 DEBUG { 17:50:05 DEBUG "status": "success", 17:50:05 DEBUG "data": { 17:50:05 DEBUG "resultType": "vector", 17:50:05 DEBUG "result": [ 17:50:05 DEBUG { 17:50:05 DEBUG "metric": { 17:50:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:50:05 DEBUG }, 17:50:05 DEBUG "value": [ 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG [...] 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG 17:50:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:50:05 DEBUG }, 17:50:05 DEBUG "value": [ 17:50:05 DEBUG 1713545405, 17:50:05 DEBUG "4200953.5" 17:50:05 DEBUG ] 17:50:05 DEBUG } 17:50:05 DEBUG ] 17:50:05 DEBUG } 17:50:05 DEBUG } 17:50:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545406" 17:50:06 INFO [http_cmd]: http status code OK 17:50:06 DEBUG --- status code --- 17:50:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:06 DEBUG --- http response --- 17:50:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:50:06 DEBUG ----- output ----- 17:50:06 DEBUG { 17:50:06 DEBUG "status": "success", 17:50:06 DEBUG "data": { 17:50:06 DEBUG "resultType": "vector", 17:50:06 DEBUG "result": [ 17:50:06 DEBUG { 17:50:06 DEBUG "metric": { 17:50:06 DEBUG "instance": "10.142.0.107:9100" 17:50:06 DEBUG }, 17:50:06 DEBUG "value": [ 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG [...] 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG "instance": "10.142.0.43:9100" 17:50:06 DEBUG }, 17:50:06 DEBUG "value": [ 17:50:06 DEBUG 1713545406, 17:50:06 DEBUG "1783484068864" 17:50:06 DEBUG ] 17:50:06 DEBUG } 17:50:06 DEBUG ] 17:50:06 DEBUG } 17:50:06 DEBUG } 17:50:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545406" 17:50:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545406" 17:50:06 INFO [http_cmd]: http status code OK 17:50:06 DEBUG --- status code --- 17:50:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:06 DEBUG --- http response --- 17:50:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:50:06 DEBUG ----- output ----- 17:50:06 DEBUG { 17:50:06 DEBUG "status": "success", 17:50:06 DEBUG "data": { 17:50:06 DEBUG "resultType": "vector", 17:50:06 DEBUG "result": [ 17:50:06 DEBUG { 17:50:06 DEBUG "metric": { 17:50:06 DEBUG "pod": "ds-idrepo-0" 17:50:06 DEBUG }, 17:50:06 DEBUG "value": [ 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG [...] 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG "pod": "ds-idrepo-2" 17:50:06 DEBUG }, 17:50:06 DEBUG "value": [ 17:50:06 DEBUG 1713545406, 17:50:06 DEBUG "0" 17:50:06 DEBUG ] 17:50:06 DEBUG } 17:50:06 DEBUG ] 17:50:06 DEBUG } 17:50:06 DEBUG } 17:50:06 INFO [http_cmd]: http status code OK 17:50:06 DEBUG --- status code --- 17:50:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:06 DEBUG --- http response --- 17:50:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:50:06 DEBUG ----- output ----- 17:50:06 DEBUG { 17:50:06 DEBUG "status": "success", 17:50:06 DEBUG "data": { 17:50:06 DEBUG "resultType": "vector", 17:50:06 DEBUG "result": [ 17:50:06 DEBUG { 17:50:06 DEBUG "metric": { 17:50:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:50:06 DEBUG }, 17:50:06 DEBUG "value": [ 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG [...] 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:50:06 DEBUG }, 17:50:06 DEBUG "value": [ 17:50:06 DEBUG 1713545406, 17:50:06 DEBUG "5000015872" 17:50:06 DEBUG ] 17:50:06 DEBUG } 17:50:06 DEBUG ] 17:50:06 DEBUG } 17:50:06 DEBUG } 17:50:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545406" 17:50:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545406" 17:50:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545406" 17:50:06 INFO [http_cmd]: http status code OK 17:50:06 DEBUG --- status code --- 17:50:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:06 DEBUG --- http response --- 17:50:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:50:06 DEBUG ----- output ----- 17:50:06 DEBUG { 17:50:06 DEBUG "status": "success", 17:50:06 DEBUG "data": { 17:50:06 DEBUG "resultType": "vector", 17:50:06 DEBUG "result": [ 17:50:06 DEBUG { 17:50:06 DEBUG "metric": { 17:50:06 DEBUG "pod": "ds-cts-0" 17:50:06 DEBUG }, 17:50:06 DEBUG "value": [ 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG [...] 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG "pod": "ds-cts-2" 17:50:06 DEBUG }, 17:50:06 DEBUG "value": [ 17:50:06 DEBUG 1713545406, 17:50:06 DEBUG "8.154933333333332" 17:50:06 DEBUG ] 17:50:06 DEBUG } 17:50:06 DEBUG ] 17:50:06 DEBUG } 17:50:06 DEBUG } 17:50:06 INFO [http_cmd]: http status code OK 17:50:06 DEBUG --- status code --- 17:50:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:06 INFO [http_cmd]: http status code OK 17:50:06 DEBUG --- http response --- 17:50:06 DEBUG --- status code --- 17:50:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:06 DEBUG --- http response --- 17:50:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:50:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:50:06 DEBUG ----- output ----- 17:50:06 DEBUG ----- output ----- 17:50:06 DEBUG { 17:50:06 DEBUG { 17:50:06 DEBUG "status": "success", 17:50:06 DEBUG "status": "success", 17:50:06 DEBUG "data": { 17:50:06 DEBUG "data": { 17:50:06 DEBUG "resultType": "vector", 17:50:06 DEBUG "resultType": "vector", 17:50:06 DEBUG "result": [ 17:50:06 DEBUG "result": [ 17:50:06 DEBUG { 17:50:06 DEBUG { 17:50:06 DEBUG "metric": { 17:50:06 DEBUG "metric": { 17:50:06 DEBUG "pod": "ds-cts-0" 17:50:06 DEBUG "pod": "ds-idrepo-0" 17:50:06 DEBUG }, 17:50:06 DEBUG }, 17:50:06 DEBUG "value": [ 17:50:06 DEBUG "value": [ 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG [...] 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG "pod": "ds-cts-2" 17:50:06 DEBUG }, 17:50:06 DEBUG "value": [ 17:50:06 DEBUG 1713545406, 17:50:06 DEBUG "27.766666666666666" 17:50:06 DEBUG ] 17:50:06 DEBUG } 17:50:06 DEBUG ] 17:50:06 DEBUG } 17:50:06 DEBUG } 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG [...] 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG 17:50:06 DEBUG "pod": "ds-idrepo-2" 17:50:06 DEBUG }, 17:50:06 DEBUG "value": [ 17:50:06 DEBUG 1713545406, 17:50:06 DEBUG "0" 17:50:06 DEBUG ] 17:50:06 DEBUG } 17:50:06 DEBUG ] 17:50:06 DEBUG } 17:50:06 DEBUG } 17:50:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545407" 17:50:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545407" 17:50:07 INFO [http_cmd]: http status code OK 17:50:07 DEBUG --- status code --- 17:50:07 INFO [http_cmd]: http status code OK 17:50:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:07 DEBUG --- status code --- 17:50:07 DEBUG --- http response --- 17:50:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:50:07 DEBUG --- http response --- 17:50:07 DEBUG ----- output ----- 17:50:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (89 lines): 17:50:07 DEBUG { 17:50:07 DEBUG ----- output ----- 17:50:07 DEBUG "status": "success", 17:50:07 DEBUG "data": { 17:50:07 DEBUG { 17:50:07 DEBUG "resultType": "vector", 17:50:07 DEBUG "status": "success", 17:50:07 DEBUG "result": [ 17:50:07 DEBUG "data": { 17:50:07 DEBUG { 17:50:07 DEBUG "resultType": "vector", 17:50:07 DEBUG "metric": { 17:50:07 DEBUG "result": [ 17:50:07 DEBUG "instance": "10.142.0.107:9100" 17:50:07 DEBUG { 17:50:07 DEBUG }, 17:50:07 DEBUG "metric": { 17:50:07 DEBUG "value": [ 17:50:07 DEBUG "pod": "ds-idrepo-0" 17:50:07 DEBUG 17:50:07 DEBUG }, 17:50:07 DEBUG 17:50:07 DEBUG "value": [ 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG [...] 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG [...] 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG "instance": "10.142.0.43:9100" 17:50:07 DEBUG 17:50:07 DEBUG }, 17:50:07 DEBUG 17:50:07 DEBUG "value": [ 17:50:07 DEBUG "pod": "ds-idrepo-2" 17:50:07 DEBUG 1713545407, 17:50:07 DEBUG }, 17:50:07 DEBUG "49460582.86666667" 17:50:07 DEBUG "value": [ 17:50:07 DEBUG ] 17:50:07 DEBUG 1713545407, 17:50:07 DEBUG } 17:50:07 DEBUG "5955.431589567075" 17:50:07 DEBUG ] 17:50:07 DEBUG } 17:50:07 DEBUG ] 17:50:07 DEBUG } 17:50:07 DEBUG } 17:50:07 DEBUG ] 17:50:07 DEBUG } 17:50:07 DEBUG } 17:50:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545407" 17:50:07 INFO [http_cmd]: http status code OK 17:50:07 DEBUG --- status code --- 17:50:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:07 DEBUG --- http response --- 17:50:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:50:07 DEBUG ----- output ----- 17:50:07 DEBUG { 17:50:07 DEBUG "status": "success", 17:50:07 DEBUG "data": { 17:50:07 DEBUG "resultType": "vector", 17:50:07 DEBUG "result": [ 17:50:07 DEBUG { 17:50:07 DEBUG "metric": { 17:50:07 DEBUG "instance": "10.142.0.107:9100" 17:50:07 DEBUG }, 17:50:07 DEBUG "value": [ 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG [...] 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG "instance": "10.142.0.43:9100" 17:50:07 DEBUG }, 17:50:07 DEBUG "value": [ 17:50:07 DEBUG 1713545407, 17:50:07 DEBUG "49454995.93333333" 17:50:07 DEBUG ] 17:50:07 DEBUG } 17:50:07 DEBUG ] 17:50:07 DEBUG } 17:50:07 DEBUG } 17:50:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545407" 17:50:07 INFO [http_cmd]: http status code OK 17:50:07 DEBUG --- status code --- 17:50:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:07 DEBUG --- http response --- 17:50:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:50:07 DEBUG ----- output ----- 17:50:07 DEBUG { 17:50:07 DEBUG "status": "success", 17:50:07 DEBUG "data": { 17:50:07 DEBUG "resultType": "vector", 17:50:07 DEBUG "result": [ 17:50:07 DEBUG { 17:50:07 DEBUG "metric": { 17:50:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:50:07 DEBUG }, 17:50:07 DEBUG "value": [ 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG [...] 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG 17:50:07 DEBUG "pod": "ds-idrepo-2" 17:50:07 DEBUG }, 17:50:07 DEBUG "value": [ 17:50:07 DEBUG 1713545407, 17:50:07 DEBUG "0" 17:50:07 DEBUG ] 17:50:07 DEBUG } 17:50:07 DEBUG ] 17:50:07 DEBUG } 17:50:07 DEBUG } 17:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545408" 17:50:08 INFO [http_cmd]: http status code OK 17:50:08 DEBUG --- status code --- 17:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:08 DEBUG --- http response --- 17:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:50:08 DEBUG ----- output ----- 17:50:08 DEBUG { 17:50:08 DEBUG "status": "success", 17:50:08 DEBUG "data": { 17:50:08 DEBUG "resultType": "vector", 17:50:08 DEBUG "result": [ 17:50:08 DEBUG { 17:50:08 DEBUG "metric": { 17:50:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:50:08 DEBUG }, 17:50:08 DEBUG "value": [ 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG [...] 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:50:08 DEBUG }, 17:50:08 DEBUG "value": [ 17:50:08 DEBUG 1713545408, 17:50:08 DEBUG "1366.9666666666667" 17:50:08 DEBUG ] 17:50:08 DEBUG } 17:50:08 DEBUG ] 17:50:08 DEBUG } 17:50:08 DEBUG } 17:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545408" 17:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545408" 17:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545408" 17:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545408" 17:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545408" 17:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545408" 17:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545408" 17:50:08 INFO [http_cmd]: http status code OK 17:50:08 DEBUG --- status code --- 17:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:08 DEBUG --- http response --- 17:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:50:08 DEBUG ----- output ----- 17:50:08 DEBUG { 17:50:08 DEBUG "status": "success", 17:50:08 DEBUG "data": { 17:50:08 DEBUG "resultType": "vector", 17:50:08 DEBUG "result": [ 17:50:08 DEBUG { 17:50:08 DEBUG "metric": { 17:50:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:50:08 DEBUG }, 17:50:08 DEBUG "value": [ 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG [...] 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:50:08 DEBUG }, 17:50:08 DEBUG "value": [ 17:50:08 DEBUG 1713545408, 17:50:08 DEBUG "0.024" 17:50:08 DEBUG ] 17:50:08 DEBUG } 17:50:08 DEBUG ] 17:50:08 DEBUG } 17:50:08 DEBUG } 17:50:08 INFO [http_cmd]: http status code OK 17:50:08 DEBUG --- status code --- 17:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:08 DEBUG --- http response --- 17:50:08 INFO [http_cmd]: http status code OK 17:50:08 DEBUG --- status code --- 17:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:08 INFO [http_cmd]: http status code OK 17:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:50:08 INFO [http_cmd]: http status code OK 17:50:08 DEBUG --- http response --- 17:50:08 DEBUG --- status code --- 17:50:08 DEBUG ----- output ----- 17:50:08 DEBUG --- status code --- 17:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:50:08 INFO [http_cmd]: http status code OK 17:50:08 DEBUG --- status code --- 17:50:08 INFO [http_cmd]: http status code OK 17:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:08 DEBUG --- http response --- 17:50:08 DEBUG ----- output ----- 17:50:08 DEBUG { 17:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:08 DEBUG --- status code --- 17:50:08 DEBUG --- http response --- 17:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:50:08 DEBUG { 17:50:08 DEBUG "status": "success", 17:50:08 DEBUG --- http response --- 17:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:50:08 DEBUG ----- output ----- 17:50:08 DEBUG "status": "success", 17:50:08 DEBUG "data": { 17:50:08 DEBUG --- http response --- 17:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:50:08 DEBUG ----- output ----- 17:50:08 DEBUG { 17:50:08 DEBUG "data": { 17:50:08 DEBUG "resultType": "vector", 17:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:50:08 DEBUG ----- output ----- 17:50:08 DEBUG { 17:50:08 DEBUG "status": "success", 17:50:08 DEBUG "resultType": "vector", 17:50:08 DEBUG "result": [ 17:50:08 DEBUG ----- output ----- 17:50:08 DEBUG { 17:50:08 DEBUG "status": "success", 17:50:08 DEBUG "data": { 17:50:08 DEBUG "result": [ 17:50:08 DEBUG { 17:50:08 DEBUG { 17:50:08 DEBUG "status": "success", 17:50:08 DEBUG "data": { 17:50:08 DEBUG "resultType": "vector", 17:50:08 DEBUG { 17:50:08 DEBUG "metric": { 17:50:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:50:08 DEBUG "data": { 17:50:08 DEBUG "resultType": "vector", 17:50:08 DEBUG "result": [ 17:50:08 DEBUG }, 17:50:08 DEBUG "status": "success", 17:50:08 DEBUG "metric": { 17:50:08 DEBUG "resultType": "vector", 17:50:08 DEBUG "result": [ 17:50:08 DEBUG { 17:50:08 DEBUG "value": [ 17:50:08 DEBUG "data": { 17:50:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:50:08 DEBUG "result": [ 17:50:08 DEBUG { 17:50:08 DEBUG "metric": { 17:50:08 DEBUG 17:50:08 DEBUG "resultType": "vector", 17:50:08 DEBUG }, 17:50:08 DEBUG { 17:50:08 DEBUG "metric": { 17:50:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:50:08 DEBUG 17:50:08 DEBUG "result": [ 17:50:08 DEBUG "value": [ 17:50:08 DEBUG "metric": { 17:50:08 DEBUG "instance": "10.142.0.107:9100" 17:50:08 DEBUG }, 17:50:08 DEBUG 17:50:08 DEBUG { 17:50:08 DEBUG 17:50:08 DEBUG "pod": "ds-idrepo-0" 17:50:08 DEBUG }, 17:50:08 DEBUG "value": [ 17:50:08 DEBUG [...] 17:50:08 DEBUG "metric": { 17:50:08 DEBUG 17:50:08 DEBUG }, 17:50:08 DEBUG "value": [ 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG "pod": "ds-cts-0" 17:50:08 DEBUG }, 17:50:08 DEBUG "value": [ 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG "value": [ 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG [...] 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG [...] 17:50:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG [...] 17:50:08 DEBUG }, 17:50:08 DEBUG "value": [ 17:50:08 DEBUG 1713545408, 17:50:08 DEBUG [...] 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG "pod": "ds-idrepo-2" 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG "1362.9" 17:50:08 DEBUG 17:50:08 DEBUG }, 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG ] 17:50:08 DEBUG [...] 17:50:08 DEBUG "value": [ 17:50:08 DEBUG "pod": "ds-idrepo-2" 17:50:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:50:08 DEBUG } 17:50:08 DEBUG ] 17:50:08 DEBUG } 17:50:08 DEBUG "instance": "10.142.0.43:9100" 17:50:08 DEBUG }, 17:50:08 DEBUG }, 17:50:08 DEBUG } 17:50:08 DEBUG 17:50:08 DEBUG 17:50:08 DEBUG 1713545408, 17:50:08 DEBUG "value": [ 17:50:08 DEBUG "value": [ 17:50:08 DEBUG 17:50:08 DEBUG }, 17:50:08 DEBUG "4.096940387526225" 17:50:08 DEBUG 1713545408, 17:50:08 DEBUG 1713545408, 17:50:08 DEBUG "pod": "ds-cts-2" 17:50:08 DEBUG "value": [ 17:50:08 DEBUG ] 17:50:08 DEBUG "0" 17:50:08 DEBUG "8.03704585369795" 17:50:08 DEBUG }, 17:50:08 DEBUG 1713545408, 17:50:08 DEBUG } 17:50:08 DEBUG ] 17:50:08 DEBUG ] 17:50:08 DEBUG "value": [ 17:50:08 DEBUG "0.9549666666667007" 17:50:08 DEBUG ] 17:50:08 DEBUG } 17:50:08 DEBUG } 17:50:08 DEBUG 1713545408, 17:50:08 DEBUG ] 17:50:08 DEBUG } 17:50:08 DEBUG ] 17:50:08 DEBUG } 17:50:08 DEBUG "318.06666666666666" 17:50:08 DEBUG } 17:50:08 DEBUG } 17:50:08 DEBUG ] 17:50:08 DEBUG } 17:50:08 DEBUG ] 17:50:08 DEBUG } 17:50:08 DEBUG ] 17:50:08 DEBUG } 17:50:08 DEBUG } 17:50:08 DEBUG ] 17:50:08 DEBUG } 17:50:08 DEBUG } 17:50:08 DEBUG } 17:50:08 DEBUG } 17:50:08 INFO 17:50:08 INFO [loop_until]: kubectl --namespace=xlou top pods 17:50:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:08 INFO [loop_until]: OK (rc = 0) 17:50:08 DEBUG --- stdout --- 17:50:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 10366m 9012Mi am-6fd554f8f7-btd4s 10237m 9064Mi am-6fd554f8f7-wxcsj 11215m 9049Mi ds-cts-0 20398m 13588Mi ds-cts-1 20427m 13778Mi ds-cts-2 20656m 13380Mi ds-idrepo-0 4071m 3424Mi ds-idrepo-1 4009m 3394Mi ds-idrepo-2 4167m 3452Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8038m 5242Mi 17:50:08 DEBUG --- stderr --- 17:50:08 DEBUG 17:50:09 INFO 17:50:09 INFO [loop_until]: kubectl --namespace=xlou top node 17:50:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:50:09 INFO [loop_until]: OK (rc = 0) 17:50:09 DEBUG --- stdout --- 17:50:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6191Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4304m 14% 6996Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 51m 0% 2697Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 10542m 35% 12335Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 20304m 68% 16579Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5245Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 10566m 35% 11965Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 8857m 29% 3785Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 11695m 39% 11303Mi 10% gke-xlou-cdm-ds-0f601751-j488 4563m 15% 6686Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 20772m 69% 18539Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 4484m 15% 7235Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 21172m 70% 15820Mi 14% gke-xlou-cdm-frontend-a8771548-i82n 8189m 51% 8736Mi 14% 17:50:09 DEBUG --- stderr --- 17:50:09 DEBUG 17:51:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545465" 17:51:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545465" 17:51:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545465" 17:51:05 ERROR [http_cmd]: ERROR 17:51:05 INFO [http_cmd]: http status code OK 17:51:05 ERROR --- status code --- 17:51:05 DEBUG --- status code --- 17:51:05 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:51:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:05 ERROR --- http response --- 17:51:05 DEBUG --- http response --- 17:51:05 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:51:05 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:51:05 INFO [http_cmd]: http status code OK 17:51:05 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:51:05 DEBUG --- status code --- 17:51:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:05 DEBUG --- http response --- 17:51:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:51:05 DEBUG ----- output ----- 17:51:05 DEBUG { 17:51:05 DEBUG "status": "success", 17:51:05 DEBUG "data": { 17:51:05 DEBUG "resultType": "vector", 17:51:05 DEBUG "result": [ 17:51:05 DEBUG { 17:51:05 DEBUG "metric": { 17:51:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG [...] 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 1713545465, 17:51:05 DEBUG "2326.1666666666665" 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG } 17:51:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545465" 17:51:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545465" 17:51:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545465" 17:51:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545465" 17:51:05 INFO [http_cmd]: http status code OK 17:51:05 DEBUG --- status code --- 17:51:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:05 DEBUG --- http response --- 17:51:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:51:05 DEBUG ----- output ----- 17:51:05 DEBUG { 17:51:05 DEBUG "status": "success", 17:51:05 DEBUG "data": { 17:51:05 DEBUG "resultType": "vector", 17:51:05 DEBUG "result": [ 17:51:05 DEBUG { 17:51:05 DEBUG "metric": { 17:51:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG [...] 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 1713545465, 17:51:05 DEBUG "0" 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG } 17:51:05 INFO [http_cmd]: http status code OK 17:51:05 DEBUG --- status code --- 17:51:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:05 DEBUG --- http response --- 17:51:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:51:05 DEBUG ----- output ----- 17:51:05 DEBUG { 17:51:05 DEBUG "status": "success", 17:51:05 DEBUG "data": { 17:51:05 DEBUG "resultType": "vector", 17:51:05 DEBUG "result": [ 17:51:05 DEBUG { 17:51:05 DEBUG "metric": { 17:51:05 DEBUG "pod": "ds-idrepo-0" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG [...] 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG "pod": "ds-cts-2" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 1713545465, 17:51:05 DEBUG "0" 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG } 17:51:05 INFO [http_cmd]: http status code OK 17:51:05 DEBUG --- status code --- 17:51:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:05 DEBUG --- http response --- 17:51:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:51:05 DEBUG ----- output ----- 17:51:05 DEBUG { 17:51:05 DEBUG "status": "success", 17:51:05 DEBUG "data": { 17:51:05 DEBUG "resultType": "vector", 17:51:05 DEBUG "result": [ 17:51:05 DEBUG { 17:51:05 DEBUG "metric": { 17:51:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG [...] 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 1713545465, 17:51:05 DEBUG "1326.7666666666667" 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG } 17:51:05 INFO [http_cmd]: http status code OK 17:51:05 DEBUG --- status code --- 17:51:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:05 DEBUG --- http response --- 17:51:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:51:05 DEBUG ----- output ----- 17:51:05 DEBUG { 17:51:05 DEBUG "status": "success", 17:51:05 DEBUG "data": { 17:51:05 DEBUG "resultType": "vector", 17:51:05 DEBUG "result": [ 17:51:05 DEBUG { 17:51:05 DEBUG "metric": { 17:51:05 DEBUG "pod": "ds-idrepo-1" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG [...] 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG "pod": "ds-idrepo-2" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 1713545465, 17:51:05 DEBUG "0" 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG } 17:51:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545465" 17:51:05 INFO [http_cmd]: http status code OK 17:51:05 DEBUG --- status code --- 17:51:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:05 DEBUG --- http response --- 17:51:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:51:05 DEBUG ----- output ----- 17:51:05 DEBUG { 17:51:05 DEBUG "status": "success", 17:51:05 DEBUG "data": { 17:51:05 DEBUG "resultType": "vector", 17:51:05 DEBUG "result": [ 17:51:05 DEBUG { 17:51:05 DEBUG "metric": { 17:51:05 DEBUG "instance": "10.142.0.107:9100" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG [...] 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG "instance": "10.142.0.43:9100" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 1713545465, 17:51:05 DEBUG "0.8796666666666662" 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG } 17:51:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545465" 17:51:05 INFO [http_cmd]: http status code OK 17:51:05 DEBUG --- status code --- 17:51:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:05 DEBUG --- http response --- 17:51:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:51:05 DEBUG ----- output ----- 17:51:05 DEBUG { 17:51:05 DEBUG "status": "success", 17:51:05 DEBUG "data": { 17:51:05 DEBUG "resultType": "vector", 17:51:05 DEBUG "result": [ 17:51:05 DEBUG { 17:51:05 DEBUG "metric": { 17:51:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG [...] 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG 17:51:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:51:05 DEBUG }, 17:51:05 DEBUG "value": [ 17:51:05 DEBUG 1713545465, 17:51:05 DEBUG "4141461.5" 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG ] 17:51:05 DEBUG } 17:51:05 DEBUG } 17:51:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545466" 17:51:06 INFO [http_cmd]: http status code OK 17:51:06 DEBUG --- status code --- 17:51:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:06 DEBUG --- http response --- 17:51:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:51:06 DEBUG ----- output ----- 17:51:06 DEBUG { 17:51:06 DEBUG "status": "success", 17:51:06 DEBUG "data": { 17:51:06 DEBUG "resultType": "vector", 17:51:06 DEBUG "result": [ 17:51:06 DEBUG { 17:51:06 DEBUG "metric": { 17:51:06 DEBUG "instance": "10.142.0.107:9100" 17:51:06 DEBUG }, 17:51:06 DEBUG "value": [ 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG [...] 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG "instance": "10.142.0.43:9100" 17:51:06 DEBUG }, 17:51:06 DEBUG "value": [ 17:51:06 DEBUG 1713545466, 17:51:06 DEBUG "1787590401024" 17:51:06 DEBUG ] 17:51:06 DEBUG } 17:51:06 DEBUG ] 17:51:06 DEBUG } 17:51:06 DEBUG } 17:51:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545466" 17:51:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545466" 17:51:06 INFO [http_cmd]: http status code OK 17:51:06 DEBUG --- status code --- 17:51:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:06 DEBUG --- http response --- 17:51:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:51:06 DEBUG ----- output ----- 17:51:06 DEBUG { 17:51:06 DEBUG "status": "success", 17:51:06 DEBUG "data": { 17:51:06 DEBUG "resultType": "vector", 17:51:06 DEBUG "result": [ 17:51:06 DEBUG { 17:51:06 DEBUG "metric": { 17:51:06 DEBUG "pod": "ds-idrepo-0" 17:51:06 DEBUG }, 17:51:06 DEBUG "value": [ 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG [...] 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG "pod": "ds-idrepo-2" 17:51:06 DEBUG }, 17:51:06 DEBUG "value": [ 17:51:06 DEBUG 1713545466, 17:51:06 DEBUG "0" 17:51:06 DEBUG ] 17:51:06 DEBUG } 17:51:06 DEBUG ] 17:51:06 DEBUG } 17:51:06 DEBUG } 17:51:06 INFO [http_cmd]: http status code OK 17:51:06 DEBUG --- status code --- 17:51:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:06 DEBUG --- http response --- 17:51:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:51:06 DEBUG ----- output ----- 17:51:06 DEBUG { 17:51:06 DEBUG "status": "success", 17:51:06 DEBUG "data": { 17:51:06 DEBUG "resultType": "vector", 17:51:06 DEBUG "result": [ 17:51:06 DEBUG { 17:51:06 DEBUG "metric": { 17:51:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:51:06 DEBUG }, 17:51:06 DEBUG "value": [ 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG [...] 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:51:06 DEBUG }, 17:51:06 DEBUG "value": [ 17:51:06 DEBUG 1713545466, 17:51:06 DEBUG "7297318912" 17:51:06 DEBUG ] 17:51:06 DEBUG } 17:51:06 DEBUG ] 17:51:06 DEBUG } 17:51:06 DEBUG } 17:51:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545466" 17:51:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545466" 17:51:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545466" 17:51:06 INFO [http_cmd]: http status code OK 17:51:06 DEBUG --- status code --- 17:51:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:06 DEBUG --- http response --- 17:51:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:51:06 DEBUG ----- output ----- 17:51:06 DEBUG { 17:51:06 DEBUG "status": "success", 17:51:06 DEBUG "data": { 17:51:06 DEBUG "resultType": "vector", 17:51:06 DEBUG "result": [ 17:51:06 DEBUG { 17:51:06 DEBUG "metric": { 17:51:06 DEBUG "pod": "ds-cts-0" 17:51:06 DEBUG }, 17:51:06 DEBUG "value": [ 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG [...] 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG "pod": "ds-cts-2" 17:51:06 DEBUG }, 17:51:06 DEBUG "value": [ 17:51:06 DEBUG 1713545466, 17:51:06 DEBUG "7.762033333333333" 17:51:06 DEBUG ] 17:51:06 DEBUG } 17:51:06 DEBUG ] 17:51:06 DEBUG } 17:51:06 DEBUG } 17:51:06 INFO [http_cmd]: http status code OK 17:51:06 DEBUG --- status code --- 17:51:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:06 DEBUG --- http response --- 17:51:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:51:06 DEBUG ----- output ----- 17:51:06 DEBUG { 17:51:06 DEBUG "status": "success", 17:51:06 DEBUG "data": { 17:51:06 DEBUG "resultType": "vector", 17:51:06 DEBUG "result": [ 17:51:06 DEBUG { 17:51:06 DEBUG "metric": { 17:51:06 DEBUG "pod": "ds-cts-0" 17:51:06 DEBUG }, 17:51:06 DEBUG "value": [ 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG [...] 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG "pod": "ds-cts-2" 17:51:06 DEBUG }, 17:51:06 DEBUG "value": [ 17:51:06 DEBUG 1713545466, 17:51:06 DEBUG "31.433333333333334" 17:51:06 DEBUG ] 17:51:06 DEBUG } 17:51:06 DEBUG ] 17:51:06 DEBUG } 17:51:06 DEBUG } 17:51:06 INFO [http_cmd]: http status code OK 17:51:06 DEBUG --- status code --- 17:51:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:06 DEBUG --- http response --- 17:51:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:51:06 DEBUG ----- output ----- 17:51:06 DEBUG { 17:51:06 DEBUG "status": "success", 17:51:06 DEBUG "data": { 17:51:06 DEBUG "resultType": "vector", 17:51:06 DEBUG "result": [ 17:51:06 DEBUG { 17:51:06 DEBUG "metric": { 17:51:06 DEBUG "pod": "ds-idrepo-0" 17:51:06 DEBUG }, 17:51:06 DEBUG "value": [ 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG [...] 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG 17:51:06 DEBUG "pod": "ds-idrepo-2" 17:51:06 DEBUG }, 17:51:06 DEBUG "value": [ 17:51:06 DEBUG 1713545466, 17:51:06 DEBUG "0" 17:51:06 DEBUG ] 17:51:06 DEBUG } 17:51:06 DEBUG ] 17:51:06 DEBUG } 17:51:06 DEBUG } 17:51:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545467" 17:51:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545467" 17:51:07 INFO [http_cmd]: http status code OK 17:51:07 DEBUG --- status code --- 17:51:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:07 DEBUG --- http response --- 17:51:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:51:07 DEBUG ----- output ----- 17:51:07 DEBUG { 17:51:07 DEBUG "status": "success", 17:51:07 DEBUG "data": { 17:51:07 DEBUG "resultType": "vector", 17:51:07 DEBUG "result": [ 17:51:07 DEBUG { 17:51:07 DEBUG "metric": { 17:51:07 DEBUG "instance": "10.142.0.107:9100" 17:51:07 DEBUG }, 17:51:07 DEBUG "value": [ 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG [...] 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG "instance": "10.142.0.43:9100" 17:51:07 DEBUG }, 17:51:07 DEBUG "value": [ 17:51:07 DEBUG 1713545467, 17:51:07 DEBUG "48298400.76666667" 17:51:07 DEBUG ] 17:51:07 DEBUG } 17:51:07 DEBUG ] 17:51:07 DEBUG } 17:51:07 DEBUG } 17:51:07 INFO [http_cmd]: http status code OK 17:51:07 DEBUG --- status code --- 17:51:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:07 DEBUG --- http response --- 17:51:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (89 lines): 17:51:07 DEBUG ----- output ----- 17:51:07 DEBUG { 17:51:07 DEBUG "status": "success", 17:51:07 DEBUG "data": { 17:51:07 DEBUG "resultType": "vector", 17:51:07 DEBUG "result": [ 17:51:07 DEBUG { 17:51:07 DEBUG "metric": { 17:51:07 DEBUG "pod": "ds-idrepo-0" 17:51:07 DEBUG }, 17:51:07 DEBUG "value": [ 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG [...] 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 17:51:07 DEBUG }, 17:51:07 DEBUG "value": [ 17:51:07 DEBUG 1713545467, 17:51:07 DEBUG "0" 17:51:07 DEBUG ] 17:51:07 DEBUG } 17:51:07 DEBUG ] 17:51:07 DEBUG } 17:51:07 DEBUG } 17:51:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545467" 17:51:07 INFO [http_cmd]: http status code OK 17:51:07 DEBUG --- status code --- 17:51:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:07 DEBUG --- http response --- 17:51:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:51:07 DEBUG ----- output ----- 17:51:07 DEBUG { 17:51:07 DEBUG "status": "success", 17:51:07 DEBUG "data": { 17:51:07 DEBUG "resultType": "vector", 17:51:07 DEBUG "result": [ 17:51:07 DEBUG { 17:51:07 DEBUG "metric": { 17:51:07 DEBUG "instance": "10.142.0.107:9100" 17:51:07 DEBUG }, 17:51:07 DEBUG "value": [ 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG [...] 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG "instance": "10.142.0.43:9100" 17:51:07 DEBUG }, 17:51:07 DEBUG "value": [ 17:51:07 DEBUG 1713545467, 17:51:07 DEBUG "48290415.4" 17:51:07 DEBUG ] 17:51:07 DEBUG } 17:51:07 DEBUG ] 17:51:07 DEBUG } 17:51:07 DEBUG } 17:51:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545467" 17:51:07 INFO [http_cmd]: http status code OK 17:51:07 DEBUG --- status code --- 17:51:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:07 DEBUG --- http response --- 17:51:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:51:07 DEBUG ----- output ----- 17:51:07 DEBUG { 17:51:07 DEBUG "status": "success", 17:51:07 DEBUG "data": { 17:51:07 DEBUG "resultType": "vector", 17:51:07 DEBUG "result": [ 17:51:07 DEBUG { 17:51:07 DEBUG "metric": { 17:51:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:51:07 DEBUG }, 17:51:07 DEBUG "value": [ 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG [...] 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG 17:51:07 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 17:51:07 DEBUG }, 17:51:07 DEBUG "value": [ 17:51:07 DEBUG 1713545467, 17:51:07 DEBUG "0" 17:51:07 DEBUG ] 17:51:07 DEBUG } 17:51:07 DEBUG ] 17:51:07 DEBUG } 17:51:07 DEBUG } 17:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545468" 17:51:08 INFO [http_cmd]: http status code OK 17:51:08 DEBUG --- status code --- 17:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:08 DEBUG --- http response --- 17:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:51:08 DEBUG ----- output ----- 17:51:08 DEBUG { 17:51:08 DEBUG "status": "success", 17:51:08 DEBUG "data": { 17:51:08 DEBUG "resultType": "vector", 17:51:08 DEBUG "result": [ 17:51:08 DEBUG { 17:51:08 DEBUG "metric": { 17:51:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:51:08 DEBUG }, 17:51:08 DEBUG "value": [ 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG [...] 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:51:08 DEBUG }, 17:51:08 DEBUG "value": [ 17:51:08 DEBUG 1713545468, 17:51:08 DEBUG "1326.5333333333333" 17:51:08 DEBUG ] 17:51:08 DEBUG } 17:51:08 DEBUG ] 17:51:08 DEBUG } 17:51:08 DEBUG } 17:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545468" 17:51:08 INFO [http_cmd]: http status code OK 17:51:08 DEBUG --- status code --- 17:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:08 DEBUG --- http response --- 17:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:51:08 DEBUG ----- output ----- 17:51:08 DEBUG { 17:51:08 DEBUG "status": "success", 17:51:08 DEBUG "data": { 17:51:08 DEBUG "resultType": "vector", 17:51:08 DEBUG "result": [ 17:51:08 DEBUG { 17:51:08 DEBUG "metric": { 17:51:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:51:08 DEBUG }, 17:51:08 DEBUG "value": [ 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG [...] 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:51:08 DEBUG }, 17:51:08 DEBUG "value": [ 17:51:08 DEBUG 1713545468, 17:51:08 DEBUG "0.024" 17:51:08 DEBUG ] 17:51:08 DEBUG } 17:51:08 DEBUG ] 17:51:08 DEBUG } 17:51:08 DEBUG } 17:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545468" 17:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545468" 17:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545468" 17:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545468" 17:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545468" 17:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545468" 17:51:08 INFO [http_cmd]: http status code OK 17:51:08 DEBUG --- status code --- 17:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:08 DEBUG --- http response --- 17:51:08 INFO [http_cmd]: http status code OK 17:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:51:08 DEBUG --- status code --- 17:51:08 DEBUG ----- output ----- 17:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:08 DEBUG { 17:51:08 DEBUG --- http response --- 17:51:08 DEBUG "status": "success", 17:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:51:08 DEBUG ----- output ----- 17:51:08 DEBUG "data": { 17:51:08 INFO [http_cmd]: http status code OK 17:51:08 DEBUG { 17:51:08 INFO [http_cmd]: http status code OK 17:51:08 INFO [http_cmd]: http status code OK 17:51:08 DEBUG "resultType": "vector", 17:51:08 INFO [http_cmd]: http status code OK 17:51:08 DEBUG --- status code --- 17:51:08 DEBUG "status": "success", 17:51:08 DEBUG --- status code --- 17:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:08 DEBUG "result": [ 17:51:08 DEBUG --- status code --- 17:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:08 DEBUG "data": { 17:51:08 DEBUG --- http response --- 17:51:08 DEBUG --- status code --- 17:51:08 DEBUG { 17:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:08 DEBUG --- http response --- 17:51:08 DEBUG "resultType": "vector", 17:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:51:08 DEBUG ----- output ----- 17:51:08 DEBUG "metric": { 17:51:08 DEBUG --- http response --- 17:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:51:08 DEBUG "result": [ 17:51:08 DEBUG { 17:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:51:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:51:08 DEBUG }, 17:51:08 DEBUG "value": [ 17:51:08 DEBUG 17:51:08 DEBUG { 17:51:08 DEBUG --- http response --- 17:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:51:08 DEBUG "status": "success", 17:51:08 DEBUG 17:51:08 DEBUG "metric": { 17:51:08 DEBUG ----- output ----- 17:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:51:08 DEBUG ----- output ----- 17:51:08 DEBUG "data": { 17:51:08 DEBUG 17:51:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:51:08 DEBUG { 17:51:08 DEBUG ----- output ----- 17:51:08 DEBUG { 17:51:08 DEBUG "resultType": "vector", 17:51:08 DEBUG [...] 17:51:08 DEBUG }, 17:51:08 DEBUG "status": "success", 17:51:08 DEBUG "status": "success", 17:51:08 DEBUG { 17:51:08 DEBUG "result": [ 17:51:08 DEBUG 17:51:08 DEBUG "value": [ 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG "data": { 17:51:08 DEBUG { 17:51:08 DEBUG 17:51:08 DEBUG "status": "success", 17:51:08 DEBUG 17:51:08 DEBUG [...] 17:51:08 DEBUG "data": { 17:51:08 DEBUG "metric": { 17:51:08 DEBUG 17:51:08 DEBUG "data": { 17:51:08 DEBUG 17:51:08 DEBUG "resultType": "vector", 17:51:08 DEBUG "pod": "ds-cts-0" 17:51:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:51:08 DEBUG "resultType": "vector", 17:51:08 DEBUG 17:51:08 DEBUG "result": [ 17:51:08 DEBUG }, 17:51:08 DEBUG }, 17:51:08 DEBUG "result": [ 17:51:08 DEBUG 17:51:08 DEBUG { 17:51:08 DEBUG "resultType": "vector", 17:51:08 DEBUG "value": [ 17:51:08 DEBUG "value": [ 17:51:08 DEBUG { 17:51:08 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 17:51:08 DEBUG "metric": { 17:51:08 DEBUG "result": [ 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG "metric": { 17:51:08 DEBUG "instance": "10.142.0.107:9100" 17:51:08 DEBUG }, 17:51:08 DEBUG { 17:51:08 DEBUG "metric": { 17:51:08 DEBUG 17:51:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:51:08 DEBUG }, 17:51:08 DEBUG "value": [ 17:51:08 DEBUG "pod": "ds-idrepo-0" 17:51:08 DEBUG 1713545468, 17:51:08 DEBUG [...] 17:51:08 DEBUG }, 17:51:08 DEBUG "value": [ 17:51:08 DEBUG 1713545468, 17:51:08 DEBUG }, 17:51:08 DEBUG "1327.8666666666666" 17:51:08 DEBUG 17:51:08 DEBUG "value": [ 17:51:08 DEBUG 17:51:08 DEBUG "15.994416132460834" 17:51:08 DEBUG "value": [ 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG ] 17:51:08 DEBUG } 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG ] 17:51:08 DEBUG ] 17:51:08 DEBUG } 17:51:08 DEBUG [...] 17:51:08 DEBUG 17:51:08 DEBUG [...] 17:51:08 DEBUG } 17:51:08 DEBUG } 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG [...] 17:51:08 DEBUG 17:51:08 DEBUG ] 17:51:08 DEBUG "pod": "ds-cts-2" 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG } 17:51:08 DEBUG }, 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG 17:51:08 DEBUG } 17:51:08 DEBUG "value": [ 17:51:08 DEBUG "pod": "ds-idrepo-2" 17:51:08 DEBUG 17:51:08 DEBUG "instance": "10.142.0.43:9100" 17:51:08 DEBUG 1713545468, 17:51:08 DEBUG }, 17:51:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:51:08 DEBUG }, 17:51:08 DEBUG "338.9" 17:51:08 DEBUG "value": [ 17:51:08 DEBUG 1713545468, 17:51:08 DEBUG "value": [ 17:51:08 DEBUG ] 17:51:08 DEBUG }, 17:51:08 DEBUG "0" 17:51:08 DEBUG 1713545468, 17:51:08 DEBUG } 17:51:08 DEBUG "value": [ 17:51:08 DEBUG ] 17:51:08 DEBUG "0.957733333333265" 17:51:08 DEBUG ] 17:51:08 DEBUG 1713545468, 17:51:08 DEBUG } 17:51:08 DEBUG ] 17:51:08 DEBUG } 17:51:08 DEBUG ] 17:51:08 DEBUG } 17:51:08 DEBUG } 17:51:08 DEBUG } 17:51:08 DEBUG ] 17:51:08 DEBUG "8.035697062408328" 17:51:08 DEBUG } 17:51:08 DEBUG } 17:51:08 DEBUG ] 17:51:08 DEBUG } 17:51:08 DEBUG } 17:51:08 DEBUG ] 17:51:08 DEBUG } 17:51:08 DEBUG } 17:51:08 INFO 17:51:08 INFO [loop_until]: kubectl --namespace=xlou top pods 17:51:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:51:09 INFO [loop_until]: OK (rc = 0) 17:51:09 DEBUG --- stdout --- 17:51:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 9753m 9011Mi am-6fd554f8f7-btd4s 9764m 9056Mi am-6fd554f8f7-wxcsj 11006m 9045Mi ds-cts-0 20439m 13695Mi ds-cts-1 20743m 13802Mi ds-cts-2 21613m 13818Mi ds-idrepo-0 3922m 3425Mi ds-idrepo-1 3818m 3393Mi ds-idrepo-2 3871m 3453Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8003m 7255Mi 17:51:09 DEBUG --- stderr --- 17:51:09 DEBUG 17:51:09 INFO 17:51:09 INFO [loop_until]: kubectl --namespace=xlou top node 17:51:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:51:09 INFO [loop_until]: OK (rc = 0) 17:51:09 DEBUG --- stdout --- 17:51:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6191Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4003m 13% 6997Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 52m 0% 2691Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 9745m 32% 12326Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 20193m 67% 16814Mi 15% gke-xlou-cdm-default-pool-1b0769c8-wpwl 148m 0% 5242Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9809m 32% 11964Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 8187m 27% 3771Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 11209m 37% 11290Mi 10% gke-xlou-cdm-ds-0f601751-j488 4336m 14% 6680Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 20632m 69% 18525Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 4303m 14% 7223Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 20509m 68% 16278Mi 14% gke-xlou-cdm-frontend-a8771548-i82n 8237m 51% 10528Mi 17% 17:51:09 DEBUG --- stderr --- 17:51:09 DEBUG 17:52:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545525" 17:52:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545525" 17:52:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545525" 17:52:05 ERROR [http_cmd]: ERROR 17:52:05 INFO [http_cmd]: http status code OK 17:52:05 ERROR --- status code --- 17:52:05 DEBUG --- status code --- 17:52:05 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:52:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:05 ERROR --- http response --- 17:52:05 DEBUG --- http response --- 17:52:05 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:52:05 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:52:05 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:52:05 INFO [http_cmd]: http status code OK 17:52:05 DEBUG --- status code --- 17:52:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:05 DEBUG --- http response --- 17:52:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:52:05 DEBUG ----- output ----- 17:52:05 DEBUG { 17:52:05 DEBUG "status": "success", 17:52:05 DEBUG "data": { 17:52:05 DEBUG "resultType": "vector", 17:52:05 DEBUG "result": [ 17:52:05 DEBUG { 17:52:05 DEBUG "metric": { 17:52:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:52:05 DEBUG }, 17:52:05 DEBUG "value": [ 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG [...] 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:52:05 DEBUG }, 17:52:05 DEBUG "value": [ 17:52:05 DEBUG 1713545525, 17:52:05 DEBUG "2338.733333333333" 17:52:05 DEBUG ] 17:52:05 DEBUG } 17:52:05 DEBUG ] 17:52:05 DEBUG } 17:52:05 DEBUG } 17:52:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545525" 17:52:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545525" 17:52:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545525" 17:52:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545525" 17:52:05 INFO [http_cmd]: http status code OK 17:52:05 INFO [http_cmd]: http status code OK 17:52:05 INFO [http_cmd]: http status code OK 17:52:05 DEBUG --- status code --- 17:52:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:05 DEBUG --- status code --- 17:52:05 DEBUG --- status code --- 17:52:05 DEBUG --- http response --- 17:52:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:52:05 DEBUG --- http response --- 17:52:05 DEBUG --- http response --- 17:52:05 INFO [http_cmd]: http status code OK 17:52:05 DEBUG ----- output ----- 17:52:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:52:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:52:05 DEBUG --- status code --- 17:52:05 DEBUG { 17:52:05 DEBUG ----- output ----- 17:52:05 DEBUG ----- output ----- 17:52:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:05 DEBUG "status": "success", 17:52:05 DEBUG { 17:52:05 DEBUG { 17:52:05 DEBUG --- http response --- 17:52:05 DEBUG "data": { 17:52:05 DEBUG "status": "success", 17:52:05 DEBUG "status": "success", 17:52:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:52:05 DEBUG "resultType": "vector", 17:52:05 DEBUG "data": { 17:52:05 DEBUG "data": { 17:52:05 DEBUG ----- output ----- 17:52:05 DEBUG "result": [ 17:52:05 DEBUG "resultType": "vector", 17:52:05 DEBUG "resultType": "vector", 17:52:05 DEBUG { 17:52:05 DEBUG { 17:52:05 DEBUG "result": [ 17:52:05 DEBUG "result": [ 17:52:05 DEBUG "status": "success", 17:52:05 DEBUG "metric": { 17:52:05 DEBUG { 17:52:05 DEBUG { 17:52:05 DEBUG "data": { 17:52:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:52:05 DEBUG "metric": { 17:52:05 DEBUG "metric": { 17:52:05 DEBUG "resultType": "vector", 17:52:05 DEBUG }, 17:52:05 DEBUG "pod": "ds-idrepo-0" 17:52:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:52:05 DEBUG "result": [ 17:52:05 DEBUG "value": [ 17:52:05 DEBUG }, 17:52:05 DEBUG }, 17:52:05 DEBUG { 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG "value": [ 17:52:05 DEBUG "metric": { 17:52:05 DEBUG "value": [ 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG "pod": "ds-idrepo-1" 17:52:05 DEBUG 17:52:05 DEBUG [...] 17:52:05 DEBUG 17:52:05 DEBUG }, 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG "value": [ 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG [...] 17:52:05 DEBUG 17:52:05 DEBUG [...] 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:52:05 DEBUG 17:52:05 DEBUG [...] 17:52:05 DEBUG "pod": "ds-cts-2" 17:52:05 DEBUG }, 17:52:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:52:05 DEBUG }, 17:52:05 DEBUG }, 17:52:05 DEBUG "value": [ 17:52:05 DEBUG 17:52:05 DEBUG "value": [ 17:52:05 DEBUG "value": [ 17:52:05 DEBUG 1713545525, 17:52:05 DEBUG 17:52:05 DEBUG 1713545525, 17:52:05 DEBUG 1713545525, 17:52:05 DEBUG "0" 17:52:05 DEBUG 17:52:05 DEBUG "1256.2333333333333" 17:52:05 DEBUG "0" 17:52:05 DEBUG ] 17:52:05 DEBUG "pod": "ds-idrepo-2" 17:52:05 DEBUG ] 17:52:05 DEBUG ] 17:52:05 DEBUG } 17:52:05 DEBUG }, 17:52:05 DEBUG } 17:52:05 DEBUG } 17:52:05 DEBUG ] 17:52:05 DEBUG "value": [ 17:52:05 DEBUG ] 17:52:05 DEBUG ] 17:52:05 DEBUG } 17:52:05 DEBUG 1713545525, 17:52:05 DEBUG } 17:52:05 DEBUG } 17:52:05 DEBUG } 17:52:05 DEBUG "0" 17:52:05 DEBUG } 17:52:05 DEBUG } 17:52:05 DEBUG ] 17:52:05 DEBUG } 17:52:05 DEBUG ] 17:52:05 DEBUG } 17:52:05 DEBUG } 17:52:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545525" 17:52:05 INFO [http_cmd]: http status code OK 17:52:05 DEBUG --- status code --- 17:52:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:05 DEBUG --- http response --- 17:52:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:52:05 DEBUG ----- output ----- 17:52:05 DEBUG { 17:52:05 DEBUG "status": "success", 17:52:05 DEBUG "data": { 17:52:05 DEBUG "resultType": "vector", 17:52:05 DEBUG "result": [ 17:52:05 DEBUG { 17:52:05 DEBUG "metric": { 17:52:05 DEBUG "instance": "10.142.0.107:9100" 17:52:05 DEBUG }, 17:52:05 DEBUG "value": [ 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG [...] 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG 17:52:05 DEBUG "instance": "10.142.0.43:9100" 17:52:05 DEBUG }, 17:52:05 DEBUG "value": [ 17:52:05 DEBUG 1713545525, 17:52:05 DEBUG "0.9296666666666586" 17:52:05 DEBUG ] 17:52:05 DEBUG } 17:52:05 DEBUG ] 17:52:05 DEBUG } 17:52:05 DEBUG } 17:52:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545526" 17:52:06 INFO [http_cmd]: http status code OK 17:52:06 DEBUG --- status code --- 17:52:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:06 DEBUG --- http response --- 17:52:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:52:06 DEBUG ----- output ----- 17:52:06 DEBUG { 17:52:06 DEBUG "status": "success", 17:52:06 DEBUG "data": { 17:52:06 DEBUG "resultType": "vector", 17:52:06 DEBUG "result": [ 17:52:06 DEBUG { 17:52:06 DEBUG "metric": { 17:52:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG [...] 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 1713545526, 17:52:06 DEBUG "4079076.5" 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG } 17:52:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545526" 17:52:06 INFO [http_cmd]: http status code OK 17:52:06 DEBUG --- status code --- 17:52:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:06 DEBUG --- http response --- 17:52:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:52:06 DEBUG ----- output ----- 17:52:06 DEBUG { 17:52:06 DEBUG "status": "success", 17:52:06 DEBUG "data": { 17:52:06 DEBUG "resultType": "vector", 17:52:06 DEBUG "result": [ 17:52:06 DEBUG { 17:52:06 DEBUG "metric": { 17:52:06 DEBUG "instance": "10.142.0.107:9100" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG [...] 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG "instance": "10.142.0.43:9100" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 1713545526, 17:52:06 DEBUG "1791765212160" 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG } 17:52:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545526" 17:52:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545526" 17:52:06 INFO [http_cmd]: http status code OK 17:52:06 DEBUG --- status code --- 17:52:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:06 DEBUG --- http response --- 17:52:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:52:06 DEBUG ----- output ----- 17:52:06 DEBUG { 17:52:06 DEBUG "status": "success", 17:52:06 DEBUG "data": { 17:52:06 DEBUG "resultType": "vector", 17:52:06 DEBUG "result": [ 17:52:06 DEBUG { 17:52:06 DEBUG "metric": { 17:52:06 DEBUG "pod": "ds-idrepo-0" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG [...] 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG "pod": "ds-idrepo-2" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 1713545526, 17:52:06 DEBUG "0" 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG } 17:52:06 INFO [http_cmd]: http status code OK 17:52:06 DEBUG --- status code --- 17:52:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:06 DEBUG --- http response --- 17:52:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:52:06 DEBUG ----- output ----- 17:52:06 DEBUG { 17:52:06 DEBUG "status": "success", 17:52:06 DEBUG "data": { 17:52:06 DEBUG "resultType": "vector", 17:52:06 DEBUG "result": [ 17:52:06 DEBUG { 17:52:06 DEBUG "metric": { 17:52:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG [...] 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 1713545526, 17:52:06 DEBUG "8211116032" 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG } 17:52:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545526" 17:52:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545526" 17:52:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545526" 17:52:06 INFO [http_cmd]: http status code OK 17:52:06 DEBUG --- status code --- 17:52:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:06 DEBUG --- http response --- 17:52:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:52:06 DEBUG ----- output ----- 17:52:06 DEBUG { 17:52:06 DEBUG "status": "success", 17:52:06 DEBUG "data": { 17:52:06 DEBUG "resultType": "vector", 17:52:06 DEBUG "result": [ 17:52:06 DEBUG { 17:52:06 DEBUG "metric": { 17:52:06 DEBUG "pod": "ds-cts-0" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG [...] 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG "pod": "ds-cts-2" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 1713545526, 17:52:06 DEBUG "6.390433333333333" 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG } 17:52:06 INFO [http_cmd]: http status code OK 17:52:06 DEBUG --- status code --- 17:52:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:06 DEBUG --- http response --- 17:52:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:52:06 DEBUG ----- output ----- 17:52:06 DEBUG { 17:52:06 DEBUG "status": "success", 17:52:06 DEBUG "data": { 17:52:06 DEBUG "resultType": "vector", 17:52:06 DEBUG "result": [ 17:52:06 DEBUG { 17:52:06 DEBUG "metric": { 17:52:06 DEBUG "pod": "ds-cts-0" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG [...] 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG "pod": "ds-cts-2" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 1713545526, 17:52:06 DEBUG "26.099999999999998" 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG } 17:52:06 INFO [http_cmd]: http status code OK 17:52:06 DEBUG --- status code --- 17:52:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:06 DEBUG --- http response --- 17:52:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:52:06 DEBUG ----- output ----- 17:52:06 DEBUG { 17:52:06 DEBUG "status": "success", 17:52:06 DEBUG "data": { 17:52:06 DEBUG "resultType": "vector", 17:52:06 DEBUG "result": [ 17:52:06 DEBUG { 17:52:06 DEBUG "metric": { 17:52:06 DEBUG "pod": "ds-idrepo-0" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG [...] 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG 17:52:06 DEBUG "pod": "ds-idrepo-2" 17:52:06 DEBUG }, 17:52:06 DEBUG "value": [ 17:52:06 DEBUG 1713545526, 17:52:06 DEBUG "0" 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG ] 17:52:06 DEBUG } 17:52:06 DEBUG } 17:52:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545527" 17:52:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545527" 17:52:07 INFO [http_cmd]: http status code OK 17:52:07 INFO [http_cmd]: http status code OK 17:52:07 DEBUG --- status code --- 17:52:07 DEBUG --- status code --- 17:52:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:07 DEBUG --- http response --- 17:52:07 DEBUG --- http response --- 17:52:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (89 lines): 17:52:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:52:07 DEBUG ----- output ----- 17:52:07 DEBUG ----- output ----- 17:52:07 DEBUG { 17:52:07 DEBUG { 17:52:07 DEBUG "status": "success", 17:52:07 DEBUG "status": "success", 17:52:07 DEBUG "data": { 17:52:07 DEBUG "data": { 17:52:07 DEBUG "resultType": "vector", 17:52:07 DEBUG "resultType": "vector", 17:52:07 DEBUG "result": [ 17:52:07 DEBUG "result": [ 17:52:07 DEBUG { 17:52:07 DEBUG "metric": { 17:52:07 DEBUG "pod": "ds-idrepo-1" 17:52:07 DEBUG }, 17:52:07 DEBUG "value": [ 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG [...] 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG "pod": "am-6fd554f8f7-btd4s" 17:52:07 DEBUG }, 17:52:07 DEBUG "value": [ 17:52:07 DEBUG 1713545527, 17:52:07 DEBUG "1033.9191619600542" 17:52:07 DEBUG ] 17:52:07 DEBUG } 17:52:07 DEBUG ] 17:52:07 DEBUG } 17:52:07 DEBUG } 17:52:07 DEBUG { 17:52:07 DEBUG "metric": { 17:52:07 DEBUG "instance": "10.142.0.107:9100" 17:52:07 DEBUG }, 17:52:07 DEBUG "value": [ 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG [...] 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG "instance": "10.142.0.43:9100" 17:52:07 DEBUG }, 17:52:07 DEBUG "value": [ 17:52:07 DEBUG 1713545527, 17:52:07 DEBUG "47232391.4" 17:52:07 DEBUG ] 17:52:07 DEBUG } 17:52:07 DEBUG ] 17:52:07 DEBUG } 17:52:07 DEBUG } 17:52:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545527" 17:52:07 INFO [http_cmd]: http status code OK 17:52:07 DEBUG --- status code --- 17:52:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:07 DEBUG --- http response --- 17:52:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:52:07 DEBUG ----- output ----- 17:52:07 DEBUG { 17:52:07 DEBUG "status": "success", 17:52:07 DEBUG "data": { 17:52:07 DEBUG "resultType": "vector", 17:52:07 DEBUG "result": [ 17:52:07 DEBUG { 17:52:07 DEBUG "metric": { 17:52:07 DEBUG "instance": "10.142.0.107:9100" 17:52:07 DEBUG }, 17:52:07 DEBUG "value": [ 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG [...] 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG "instance": "10.142.0.43:9100" 17:52:07 DEBUG }, 17:52:07 DEBUG "value": [ 17:52:07 DEBUG 1713545527, 17:52:07 DEBUG "47224401.8" 17:52:07 DEBUG ] 17:52:07 DEBUG } 17:52:07 DEBUG ] 17:52:07 DEBUG } 17:52:07 DEBUG } 17:52:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545527" 17:52:07 INFO [http_cmd]: http status code OK 17:52:07 DEBUG --- status code --- 17:52:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:07 DEBUG --- http response --- 17:52:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:52:07 DEBUG ----- output ----- 17:52:07 DEBUG { 17:52:07 DEBUG "status": "success", 17:52:07 DEBUG "data": { 17:52:07 DEBUG "resultType": "vector", 17:52:07 DEBUG "result": [ 17:52:07 DEBUG { 17:52:07 DEBUG "metric": { 17:52:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:52:07 DEBUG }, 17:52:07 DEBUG "value": [ 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG [...] 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG 17:52:07 DEBUG "pod": "am-6fd554f8f7-btd4s" 17:52:07 DEBUG }, 17:52:07 DEBUG "value": [ 17:52:07 DEBUG 1713545527, 17:52:07 DEBUG "0" 17:52:07 DEBUG ] 17:52:07 DEBUG } 17:52:07 DEBUG ] 17:52:07 DEBUG } 17:52:07 DEBUG } 17:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545528" 17:52:08 INFO [http_cmd]: http status code OK 17:52:08 DEBUG --- status code --- 17:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:08 DEBUG --- http response --- 17:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:52:08 DEBUG ----- output ----- 17:52:08 DEBUG { 17:52:08 DEBUG "status": "success", 17:52:08 DEBUG "data": { 17:52:08 DEBUG "resultType": "vector", 17:52:08 DEBUG "result": [ 17:52:08 DEBUG { 17:52:08 DEBUG "metric": { 17:52:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:52:08 DEBUG }, 17:52:08 DEBUG "value": [ 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG [...] 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:52:08 DEBUG }, 17:52:08 DEBUG "value": [ 17:52:08 DEBUG 1713545528, 17:52:08 DEBUG "1256.0666666666666" 17:52:08 DEBUG ] 17:52:08 DEBUG } 17:52:08 DEBUG ] 17:52:08 DEBUG } 17:52:08 DEBUG } 17:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545528" 17:52:08 INFO [http_cmd]: http status code OK 17:52:08 DEBUG --- status code --- 17:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:08 DEBUG --- http response --- 17:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:52:08 DEBUG ----- output ----- 17:52:08 DEBUG { 17:52:08 DEBUG "status": "success", 17:52:08 DEBUG "data": { 17:52:08 DEBUG "resultType": "vector", 17:52:08 DEBUG "result": [ 17:52:08 DEBUG { 17:52:08 DEBUG "metric": { 17:52:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:52:08 DEBUG }, 17:52:08 DEBUG "value": [ 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG [...] 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:52:08 DEBUG }, 17:52:08 DEBUG "value": [ 17:52:08 DEBUG 1713545528, 17:52:08 DEBUG "0.024" 17:52:08 DEBUG ] 17:52:08 DEBUG } 17:52:08 DEBUG ] 17:52:08 DEBUG } 17:52:08 DEBUG } 17:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545528" 17:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545528" 17:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545528" 17:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545528" 17:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545528" 17:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545528" 17:52:08 INFO [http_cmd]: http status code OK 17:52:08 DEBUG --- status code --- 17:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:08 DEBUG --- http response --- 17:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:52:08 DEBUG ----- output ----- 17:52:08 DEBUG { 17:52:08 INFO [http_cmd]: http status code OK 17:52:08 DEBUG "status": "success", 17:52:08 DEBUG --- status code --- 17:52:08 DEBUG "data": { 17:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:08 DEBUG "resultType": "vector", 17:52:08 INFO [http_cmd]: http status code OK 17:52:08 DEBUG --- http response --- 17:52:08 INFO [http_cmd]: http status code OK 17:52:08 DEBUG "result": [ 17:52:08 INFO [http_cmd]: http status code OK 17:52:08 DEBUG --- status code --- 17:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:52:08 DEBUG --- status code --- 17:52:08 DEBUG { 17:52:08 DEBUG --- status code --- 17:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:08 DEBUG ----- output ----- 17:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:08 DEBUG "metric": { 17:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:08 DEBUG --- http response --- 17:52:08 DEBUG { 17:52:08 DEBUG --- http response --- 17:52:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:52:08 DEBUG --- http response --- 17:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:52:08 DEBUG "status": "success", 17:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:52:08 DEBUG }, 17:52:08 INFO [http_cmd]: http status code OK 17:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:52:08 DEBUG ----- output ----- 17:52:08 DEBUG "data": { 17:52:08 DEBUG ----- output ----- 17:52:08 DEBUG "value": [ 17:52:08 DEBUG --- status code --- 17:52:08 DEBUG ----- output ----- 17:52:08 DEBUG { 17:52:08 DEBUG "resultType": "vector", 17:52:08 DEBUG { 17:52:08 DEBUG 17:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:52:08 DEBUG { 17:52:08 DEBUG "status": "success", 17:52:08 DEBUG "result": [ 17:52:08 DEBUG "status": "success", 17:52:08 DEBUG 17:52:08 DEBUG --- http response --- 17:52:08 DEBUG "status": "success", 17:52:08 DEBUG "data": { 17:52:08 DEBUG { 17:52:08 DEBUG "data": { 17:52:08 DEBUG 17:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:52:08 DEBUG "data": { 17:52:08 DEBUG "resultType": "vector", 17:52:08 DEBUG "metric": { 17:52:08 DEBUG "resultType": "vector", 17:52:08 DEBUG [...] 17:52:08 DEBUG ----- output ----- 17:52:08 DEBUG "resultType": "vector", 17:52:08 DEBUG "result": [ 17:52:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:52:08 DEBUG "result": [ 17:52:08 DEBUG 17:52:08 DEBUG { 17:52:08 DEBUG "result": [ 17:52:08 DEBUG { 17:52:08 DEBUG }, 17:52:08 DEBUG { 17:52:08 DEBUG 17:52:08 DEBUG "status": "success", 17:52:08 DEBUG { 17:52:08 DEBUG "metric": { 17:52:08 DEBUG "value": [ 17:52:08 DEBUG "metric": { 17:52:08 DEBUG 17:52:08 DEBUG "data": { 17:52:08 DEBUG "metric": { 17:52:08 DEBUG "pod": "ds-cts-0" 17:52:08 DEBUG 17:52:08 DEBUG "instance": "10.142.0.107:9100" 17:52:08 DEBUG "pod": "ds-idrepo-2" 17:52:08 DEBUG "resultType": "vector", 17:52:08 DEBUG "pod": "ds-idrepo-0" 17:52:08 DEBUG }, 17:52:08 DEBUG 17:52:08 DEBUG }, 17:52:08 DEBUG }, 17:52:08 DEBUG "result": [ 17:52:08 DEBUG }, 17:52:08 DEBUG "value": [ 17:52:08 DEBUG 17:52:08 DEBUG "value": [ 17:52:08 DEBUG "value": [ 17:52:08 DEBUG 1713545528, 17:52:08 DEBUG "value": [ 17:52:08 DEBUG 17:52:08 DEBUG [...] 17:52:08 DEBUG 17:52:08 DEBUG { 17:52:08 DEBUG "0" 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG "metric": { 17:52:08 DEBUG ] 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:52:08 DEBUG } 17:52:08 DEBUG 17:52:08 DEBUG [...] 17:52:08 DEBUG 17:52:08 DEBUG [...] 17:52:08 DEBUG }, 17:52:08 DEBUG ] 17:52:08 DEBUG [...] 17:52:08 DEBUG 17:52:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:52:08 DEBUG 17:52:08 DEBUG "value": [ 17:52:08 DEBUG } 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG }, 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG } 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG "value": [ 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG 17:52:08 DEBUG "pod": "ds-idrepo-2" 17:52:08 DEBUG }, 17:52:08 DEBUG "instance": "10.142.0.43:9100" 17:52:08 DEBUG 17:52:08 DEBUG "pod": "ds-cts-2" 17:52:08 DEBUG "value": [ 17:52:08 DEBUG 1713545528, 17:52:08 DEBUG }, 17:52:08 DEBUG [...] 17:52:08 DEBUG }, 17:52:08 DEBUG 1713545528, 17:52:08 DEBUG "1255.2666666666667" 17:52:08 DEBUG "value": [ 17:52:08 DEBUG 17:52:08 DEBUG "value": [ 17:52:08 DEBUG "0" 17:52:08 DEBUG ] 17:52:08 DEBUG 1713545528, 17:52:08 DEBUG 17:52:08 DEBUG 1713545528, 17:52:08 DEBUG ] 17:52:08 DEBUG } 17:52:08 DEBUG "0.9591000000000889" 17:52:08 DEBUG 17:52:08 DEBUG "362.6666666666667" 17:52:08 DEBUG ] 17:52:08 DEBUG ] 17:52:08 DEBUG ] 17:52:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:52:08 DEBUG } 17:52:08 DEBUG } 17:52:08 DEBUG } 17:52:08 DEBUG } 17:52:08 DEBUG }, 17:52:08 DEBUG ] 17:52:08 DEBUG ] 17:52:08 DEBUG } 17:52:08 DEBUG ] 17:52:08 DEBUG "value": [ 17:52:08 DEBUG } 17:52:08 DEBUG } 17:52:08 DEBUG } 17:52:08 DEBUG 1713545528, 17:52:08 DEBUG } 17:52:08 DEBUG } 17:52:08 DEBUG } 17:52:08 DEBUG "8.020820393070307" 17:52:08 DEBUG ] 17:52:08 DEBUG } 17:52:08 DEBUG ] 17:52:08 DEBUG } 17:52:08 DEBUG } 17:52:09 INFO 17:52:09 INFO [loop_until]: kubectl --namespace=xlou top pods 17:52:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:52:09 INFO [loop_until]: OK (rc = 0) 17:52:09 DEBUG --- stdout --- 17:52:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 9456m 9012Mi am-6fd554f8f7-btd4s 9552m 9058Mi am-6fd554f8f7-wxcsj 10166m 9044Mi ds-cts-0 20228m 13272Mi ds-cts-1 20705m 13045Mi ds-cts-2 20922m 13713Mi ds-idrepo-0 3666m 3424Mi ds-idrepo-1 3796m 3394Mi ds-idrepo-2 3842m 3453Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8016m 8152Mi 17:52:09 DEBUG --- stderr --- 17:52:09 DEBUG 17:52:09 INFO 17:52:09 INFO [loop_until]: kubectl --namespace=xlou top node 17:52:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:52:09 INFO [loop_until]: OK (rc = 0) 17:52:09 DEBUG --- stdout --- 17:52:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6181Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 3935m 13% 7001Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 46m 0% 2680Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 9499m 31% 12327Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 19574m 65% 16451Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5248Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9561m 32% 11960Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 7739m 25% 3754Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 10581m 35% 11286Mi 10% gke-xlou-cdm-ds-0f601751-j488 4206m 14% 6683Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 20554m 68% 17732Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 4143m 13% 7236Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 20837m 69% 15831Mi 14% gke-xlou-cdm-frontend-a8771548-i82n 8249m 51% 11164Mi 19% 17:52:09 DEBUG --- stderr --- 17:52:09 DEBUG 17:52:21 WARNING File /tmp/lodemon_data-20240419_165801/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [19/Apr/2024 17:52:21] "GET /monitoring/average?start_time=20240419_163217&stop_time=20240419_165021 HTTP/1.1" 200 - 17:53:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545585" 17:53:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545585" 17:53:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545585" 17:53:05 INFO [http_cmd]: http status code OK 17:53:05 INFO [http_cmd]: http status code OK 17:53:05 DEBUG --- status code --- 17:53:05 DEBUG --- status code --- 17:53:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:05 DEBUG --- http response --- 17:53:05 ERROR [http_cmd]: ERROR 17:53:05 DEBUG --- http response --- 17:53:05 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:53:05 ERROR --- status code --- 17:53:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:53:05 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:53:05 DEBUG ----- output ----- 17:53:05 ERROR --- http response --- 17:53:05 DEBUG { 17:53:05 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:53:05 DEBUG "status": "success", 17:53:05 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:53:05 DEBUG "data": { 17:53:05 DEBUG "resultType": "vector", 17:53:05 DEBUG "result": [ 17:53:05 DEBUG { 17:53:05 DEBUG "metric": { 17:53:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:53:05 DEBUG }, 17:53:05 DEBUG "value": [ 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG [...] 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:53:05 DEBUG }, 17:53:05 DEBUG "value": [ 17:53:05 DEBUG 1713545585, 17:53:05 DEBUG "2337.366666666667" 17:53:05 DEBUG ] 17:53:05 DEBUG } 17:53:05 DEBUG ] 17:53:05 DEBUG } 17:53:05 DEBUG } 17:53:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545585" 17:53:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545585" 17:53:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545585" 17:53:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545585" 17:53:05 INFO [http_cmd]: http status code OK 17:53:05 INFO [http_cmd]: http status code OK 17:53:05 DEBUG --- status code --- 17:53:05 DEBUG --- status code --- 17:53:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:05 DEBUG --- http response --- 17:53:05 DEBUG --- http response --- 17:53:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:53:05 INFO [http_cmd]: http status code OK 17:53:05 INFO [http_cmd]: http status code OK 17:53:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:53:05 DEBUG ----- output ----- 17:53:05 DEBUG --- status code --- 17:53:05 DEBUG --- status code --- 17:53:05 DEBUG ----- output ----- 17:53:05 DEBUG { 17:53:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:05 DEBUG { 17:53:05 DEBUG "status": "success", 17:53:05 DEBUG --- http response --- 17:53:05 DEBUG --- http response --- 17:53:05 DEBUG "status": "success", 17:53:05 DEBUG "data": { 17:53:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:53:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:53:05 DEBUG "data": { 17:53:05 DEBUG "resultType": "vector", 17:53:05 DEBUG ----- output ----- 17:53:05 DEBUG ----- output ----- 17:53:05 DEBUG "resultType": "vector", 17:53:05 DEBUG "result": [ 17:53:05 DEBUG { 17:53:05 DEBUG { 17:53:05 DEBUG "result": [ 17:53:05 DEBUG { 17:53:05 DEBUG "status": "success", 17:53:05 DEBUG "status": "success", 17:53:05 DEBUG { 17:53:05 DEBUG "metric": { 17:53:05 DEBUG "data": { 17:53:05 DEBUG "data": { 17:53:05 DEBUG "metric": { 17:53:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:53:05 DEBUG "resultType": "vector", 17:53:05 DEBUG "resultType": "vector", 17:53:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:53:05 DEBUG }, 17:53:05 DEBUG "result": [ 17:53:05 DEBUG "result": [ 17:53:05 DEBUG }, 17:53:05 DEBUG "value": [ 17:53:05 DEBUG { 17:53:05 DEBUG { 17:53:05 DEBUG "value": [ 17:53:05 DEBUG 17:53:05 DEBUG "metric": { 17:53:05 DEBUG "metric": { 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG "pod": "ds-idrepo-0" 17:53:05 DEBUG "pod": "ds-idrepo-0" 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG }, 17:53:05 DEBUG }, 17:53:05 DEBUG 17:53:05 DEBUG [...] 17:53:05 DEBUG "value": [ 17:53:05 DEBUG "value": [ 17:53:05 DEBUG [...] 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:53:05 DEBUG [...] 17:53:05 DEBUG [...] 17:53:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:53:05 DEBUG }, 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG }, 17:53:05 DEBUG "value": [ 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG "value": [ 17:53:05 DEBUG 1713545585, 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 1713545585, 17:53:05 DEBUG "0" 17:53:05 DEBUG "pod": "ds-cts-2" 17:53:05 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 17:53:05 DEBUG "0" 17:53:05 DEBUG ] 17:53:05 DEBUG }, 17:53:05 DEBUG }, 17:53:05 DEBUG ] 17:53:05 DEBUG } 17:53:05 DEBUG "value": [ 17:53:05 DEBUG "value": [ 17:53:05 DEBUG } 17:53:05 DEBUG ] 17:53:05 DEBUG 1713545585, 17:53:05 DEBUG 1713545585, 17:53:05 DEBUG ] 17:53:05 DEBUG } 17:53:05 DEBUG "0" 17:53:05 DEBUG "0" 17:53:05 DEBUG } 17:53:05 DEBUG } 17:53:05 DEBUG ] 17:53:05 DEBUG ] 17:53:05 DEBUG } 17:53:05 DEBUG } 17:53:05 DEBUG } 17:53:05 DEBUG ] 17:53:05 DEBUG ] 17:53:05 DEBUG } 17:53:05 DEBUG } 17:53:05 DEBUG } 17:53:05 DEBUG } 17:53:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545585" 17:53:05 INFO [http_cmd]: http status code OK 17:53:05 DEBUG --- status code --- 17:53:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:05 DEBUG --- http response --- 17:53:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:53:05 DEBUG ----- output ----- 17:53:05 DEBUG { 17:53:05 DEBUG "status": "success", 17:53:05 DEBUG "data": { 17:53:05 DEBUG "resultType": "vector", 17:53:05 DEBUG "result": [ 17:53:05 DEBUG { 17:53:05 DEBUG "metric": { 17:53:05 DEBUG "instance": "10.142.0.107:9100" 17:53:05 DEBUG }, 17:53:05 DEBUG "value": [ 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG [...] 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG 17:53:05 DEBUG "instance": "10.142.0.43:9100" 17:53:05 DEBUG }, 17:53:05 DEBUG "value": [ 17:53:05 DEBUG 1713545585, 17:53:05 DEBUG "0.5703333333333357" 17:53:05 DEBUG ] 17:53:05 DEBUG } 17:53:05 DEBUG ] 17:53:05 DEBUG } 17:53:05 DEBUG } 17:53:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545586" 17:53:06 INFO [http_cmd]: http status code OK 17:53:06 DEBUG --- status code --- 17:53:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:06 DEBUG --- http response --- 17:53:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:53:06 DEBUG ----- output ----- 17:53:06 DEBUG { 17:53:06 DEBUG "status": "success", 17:53:06 DEBUG "data": { 17:53:06 DEBUG "resultType": "vector", 17:53:06 DEBUG "result": [ 17:53:06 DEBUG { 17:53:06 DEBUG "metric": { 17:53:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:53:06 DEBUG }, 17:53:06 DEBUG "value": [ 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG [...] 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:53:06 DEBUG }, 17:53:06 DEBUG "value": [ 17:53:06 DEBUG 1713545586, 17:53:06 DEBUG "3982335.5" 17:53:06 DEBUG ] 17:53:06 DEBUG } 17:53:06 DEBUG ] 17:53:06 DEBUG } 17:53:06 DEBUG } 17:53:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545586" 17:53:06 INFO [http_cmd]: http status code OK 17:53:06 DEBUG --- status code --- 17:53:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:06 DEBUG --- http response --- 17:53:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:53:06 DEBUG ----- output ----- 17:53:06 DEBUG { 17:53:06 DEBUG "status": "success", 17:53:06 DEBUG "data": { 17:53:06 DEBUG "resultType": "vector", 17:53:06 DEBUG "result": [ 17:53:06 DEBUG { 17:53:06 DEBUG "metric": { 17:53:06 DEBUG "instance": "10.142.0.107:9100" 17:53:06 DEBUG }, 17:53:06 DEBUG "value": [ 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG [...] 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG "instance": "10.142.0.43:9100" 17:53:06 DEBUG }, 17:53:06 DEBUG "value": [ 17:53:06 DEBUG 1713545586, 17:53:06 DEBUG "1795803843584" 17:53:06 DEBUG ] 17:53:06 DEBUG } 17:53:06 DEBUG ] 17:53:06 DEBUG } 17:53:06 DEBUG } 17:53:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545586" 17:53:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545586" 17:53:06 INFO [http_cmd]: http status code OK 17:53:06 DEBUG --- status code --- 17:53:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:06 DEBUG --- http response --- 17:53:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:53:06 DEBUG ----- output ----- 17:53:06 DEBUG { 17:53:06 DEBUG "status": "success", 17:53:06 DEBUG "data": { 17:53:06 DEBUG "resultType": "vector", 17:53:06 DEBUG "result": [ 17:53:06 DEBUG { 17:53:06 DEBUG "metric": { 17:53:06 DEBUG "pod": "ds-idrepo-0" 17:53:06 DEBUG }, 17:53:06 DEBUG "value": [ 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG [...] 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG "pod": "ds-idrepo-2" 17:53:06 DEBUG }, 17:53:06 DEBUG "value": [ 17:53:06 DEBUG 1713545586, 17:53:06 DEBUG "0" 17:53:06 DEBUG ] 17:53:06 DEBUG } 17:53:06 DEBUG ] 17:53:06 DEBUG } 17:53:06 DEBUG } 17:53:06 INFO [http_cmd]: http status code OK 17:53:06 DEBUG --- status code --- 17:53:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:06 DEBUG --- http response --- 17:53:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:53:06 DEBUG ----- output ----- 17:53:06 DEBUG { 17:53:06 DEBUG "status": "success", 17:53:06 DEBUG "data": { 17:53:06 DEBUG "resultType": "vector", 17:53:06 DEBUG "result": [ 17:53:06 DEBUG { 17:53:06 DEBUG "metric": { 17:53:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:53:06 DEBUG }, 17:53:06 DEBUG "value": [ 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG [...] 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:53:06 DEBUG }, 17:53:06 DEBUG "value": [ 17:53:06 DEBUG 1713545586, 17:53:06 DEBUG "117325824" 17:53:06 DEBUG ] 17:53:06 DEBUG } 17:53:06 DEBUG ] 17:53:06 DEBUG } 17:53:06 DEBUG } 17:53:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545586" 17:53:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545586" 17:53:06 INFO [http_cmd]: http status code OK 17:53:06 INFO [http_cmd]: http status code OK 17:53:06 DEBUG --- status code --- 17:53:06 DEBUG --- status code --- 17:53:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:06 DEBUG --- http response --- 17:53:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:53:06 DEBUG ----- output ----- 17:53:06 DEBUG { 17:53:06 DEBUG "status": "success", 17:53:06 DEBUG "data": { 17:53:06 DEBUG "resultType": "vector", 17:53:06 DEBUG "result": [ 17:53:06 DEBUG { 17:53:06 DEBUG "metric": { 17:53:06 DEBUG "pod": "ds-cts-0" 17:53:06 DEBUG }, 17:53:06 DEBUG "value": [ 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG [...] 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG "pod": "ds-cts-2" 17:53:06 DEBUG }, 17:53:06 DEBUG "value": [ 17:53:06 DEBUG 1713545586, 17:53:06 DEBUG "31.566666666666666" 17:53:06 DEBUG ] 17:53:06 DEBUG } 17:53:06 DEBUG ] 17:53:06 DEBUG } 17:53:06 DEBUG } 17:53:06 DEBUG --- http response --- 17:53:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:53:06 DEBUG ----- output ----- 17:53:06 DEBUG { 17:53:06 DEBUG "status": "success", 17:53:06 DEBUG "data": { 17:53:06 DEBUG "resultType": "vector", 17:53:06 DEBUG "result": [ 17:53:06 DEBUG { 17:53:06 DEBUG "metric": { 17:53:06 DEBUG "pod": "ds-cts-0" 17:53:06 DEBUG }, 17:53:06 DEBUG "value": [ 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG [...] 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG 17:53:06 DEBUG "pod": "ds-cts-2" 17:53:06 DEBUG }, 17:53:06 DEBUG "value": [ 17:53:06 DEBUG 1713545586, 17:53:06 DEBUG "3.7507" 17:53:06 DEBUG ] 17:53:06 DEBUG } 17:53:06 DEBUG ] 17:53:06 DEBUG } 17:53:06 DEBUG } 17:53:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545587" 17:53:07 INFO [http_cmd]: http status code OK 17:53:07 DEBUG --- status code --- 17:53:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:07 DEBUG --- http response --- 17:53:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:53:07 DEBUG ----- output ----- 17:53:07 DEBUG { 17:53:07 DEBUG "status": "success", 17:53:07 DEBUG "data": { 17:53:07 DEBUG "resultType": "vector", 17:53:07 DEBUG "result": [ 17:53:07 DEBUG { 17:53:07 DEBUG "metric": { 17:53:07 DEBUG "pod": "ds-idrepo-0" 17:53:07 DEBUG }, 17:53:07 DEBUG "value": [ 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG [...] 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG "pod": "ds-idrepo-2" 17:53:07 DEBUG }, 17:53:07 DEBUG "value": [ 17:53:07 DEBUG 1713545587, 17:53:07 DEBUG "0" 17:53:07 DEBUG ] 17:53:07 DEBUG } 17:53:07 DEBUG ] 17:53:07 DEBUG } 17:53:07 DEBUG } 17:53:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545587" 17:53:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545587" 17:53:07 INFO [http_cmd]: http status code OK 17:53:07 INFO [http_cmd]: http status code OK 17:53:07 DEBUG --- status code --- 17:53:07 DEBUG --- status code --- 17:53:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:07 DEBUG --- http response --- 17:53:07 DEBUG --- http response --- 17:53:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:53:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:53:07 DEBUG ----- output ----- 17:53:07 DEBUG ----- output ----- 17:53:07 DEBUG { 17:53:07 DEBUG { 17:53:07 DEBUG "status": "success", 17:53:07 DEBUG "status": "success", 17:53:07 DEBUG "data": { 17:53:07 DEBUG "data": { 17:53:07 DEBUG "resultType": "vector", 17:53:07 DEBUG "result": [ 17:53:07 DEBUG "resultType": "vector", 17:53:07 DEBUG { 17:53:07 DEBUG "result": [ 17:53:07 DEBUG "metric": { 17:53:07 DEBUG { 17:53:07 DEBUG "instance": "10.142.0.107:9100" 17:53:07 DEBUG "metric": { 17:53:07 DEBUG }, 17:53:07 DEBUG "pod": "ds-idrepo-0" 17:53:07 DEBUG "value": [ 17:53:07 DEBUG }, 17:53:07 DEBUG 17:53:07 DEBUG "value": [ 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG [...] 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG [...] 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG "instance": "10.142.0.43:9100" 17:53:07 DEBUG 17:53:07 DEBUG }, 17:53:07 DEBUG "pod": "ds-cts-2" 17:53:07 DEBUG "value": [ 17:53:07 DEBUG }, 17:53:07 DEBUG 1713545587, 17:53:07 DEBUG "value": [ 17:53:07 DEBUG "17423786.8" 17:53:07 DEBUG 1713545587, 17:53:07 DEBUG ] 17:53:07 DEBUG "58771402.495347075" 17:53:07 DEBUG } 17:53:07 DEBUG ] 17:53:07 DEBUG ] 17:53:07 DEBUG } 17:53:07 DEBUG } 17:53:07 DEBUG ] 17:53:07 DEBUG } 17:53:07 DEBUG } 17:53:07 DEBUG } 17:53:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545587" 17:53:07 INFO [http_cmd]: http status code OK 17:53:07 DEBUG --- status code --- 17:53:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:07 DEBUG --- http response --- 17:53:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:53:07 DEBUG ----- output ----- 17:53:07 DEBUG { 17:53:07 DEBUG "status": "success", 17:53:07 DEBUG "data": { 17:53:07 DEBUG "resultType": "vector", 17:53:07 DEBUG "result": [ 17:53:07 DEBUG { 17:53:07 DEBUG "metric": { 17:53:07 DEBUG "instance": "10.142.0.107:9100" 17:53:07 DEBUG }, 17:53:07 DEBUG "value": [ 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG [...] 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG "instance": "10.142.0.43:9100" 17:53:07 DEBUG }, 17:53:07 DEBUG "value": [ 17:53:07 DEBUG 1713545587, 17:53:07 DEBUG "17197114.166666664" 17:53:07 DEBUG ] 17:53:07 DEBUG } 17:53:07 DEBUG ] 17:53:07 DEBUG } 17:53:07 DEBUG } 17:53:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545587" 17:53:07 INFO [http_cmd]: http status code OK 17:53:07 DEBUG --- status code --- 17:53:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:07 DEBUG --- http response --- 17:53:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:53:07 DEBUG ----- output ----- 17:53:07 DEBUG { 17:53:07 DEBUG "status": "success", 17:53:07 DEBUG "data": { 17:53:07 DEBUG "resultType": "vector", 17:53:07 DEBUG "result": [ 17:53:07 DEBUG { 17:53:07 DEBUG "metric": { 17:53:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:53:07 DEBUG }, 17:53:07 DEBUG "value": [ 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG [...] 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG 17:53:07 DEBUG "pod": "ds-cts-2" 17:53:07 DEBUG }, 17:53:07 DEBUG "value": [ 17:53:07 DEBUG 1713545587, 17:53:07 DEBUG "143567242.29682222" 17:53:07 DEBUG ] 17:53:07 DEBUG } 17:53:07 DEBUG ] 17:53:07 DEBUG } 17:53:07 DEBUG } 17:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545588" 17:53:08 INFO [http_cmd]: http status code OK 17:53:08 DEBUG --- status code --- 17:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:08 DEBUG --- http response --- 17:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:53:08 DEBUG ----- output ----- 17:53:08 DEBUG { 17:53:08 DEBUG "status": "success", 17:53:08 DEBUG "data": { 17:53:08 DEBUG "resultType": "vector", 17:53:08 DEBUG "result": [ 17:53:08 DEBUG { 17:53:08 DEBUG "metric": { 17:53:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG [...] 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 1713545588, 17:53:08 DEBUG "0" 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG } 17:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545588" 17:53:08 INFO [http_cmd]: http status code OK 17:53:08 DEBUG --- status code --- 17:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:08 DEBUG --- http response --- 17:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:53:08 DEBUG ----- output ----- 17:53:08 DEBUG { 17:53:08 DEBUG "status": "success", 17:53:08 DEBUG "data": { 17:53:08 DEBUG "resultType": "vector", 17:53:08 DEBUG "result": [ 17:53:08 DEBUG { 17:53:08 DEBUG "metric": { 17:53:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG [...] 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 1713545588, 17:53:08 DEBUG "0.024" 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG } 17:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545588" 17:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545588" 17:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545588" 17:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545588" 17:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545588" 17:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545588" 17:53:08 INFO [http_cmd]: http status code OK 17:53:08 DEBUG --- status code --- 17:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:08 DEBUG --- http response --- 17:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 17:53:08 DEBUG ----- output ----- 17:53:08 DEBUG { 17:53:08 DEBUG "status": "success", 17:53:08 DEBUG "data": { 17:53:08 DEBUG "resultType": "vector", 17:53:08 DEBUG "result": [ 17:53:08 DEBUG { 17:53:08 DEBUG "metric": { 17:53:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG [...] 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG "pod": "ds-cts-2" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 1713545588, 17:53:08 DEBUG "20.588639243124078" 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG } 17:53:08 INFO [http_cmd]: http status code OK 17:53:08 DEBUG --- status code --- 17:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:08 DEBUG --- http response --- 17:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:53:08 DEBUG ----- output ----- 17:53:08 DEBUG { 17:53:08 DEBUG "status": "success", 17:53:08 DEBUG "data": { 17:53:08 DEBUG "resultType": "vector", 17:53:08 DEBUG "result": [ 17:53:08 DEBUG { 17:53:08 DEBUG "metric": { 17:53:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG [...] 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 1713545588, 17:53:08 DEBUG "0" 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG } 17:53:08 INFO [http_cmd]: http status code OK 17:53:08 DEBUG --- status code --- 17:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:08 DEBUG --- http response --- 17:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:53:08 DEBUG ----- output ----- 17:53:08 DEBUG { 17:53:08 DEBUG "status": "success", 17:53:08 DEBUG "data": { 17:53:08 DEBUG "resultType": "vector", 17:53:08 DEBUG "result": [ 17:53:08 DEBUG { 17:53:08 DEBUG "metric": { 17:53:08 DEBUG "instance": "10.142.0.107:9100" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG [...] 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG "instance": "10.142.0.43:9100" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 1713545588, 17:53:08 DEBUG "0.7566666666667041" 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG } 17:53:08 INFO [http_cmd]: http status code OK 17:53:08 DEBUG --- status code --- 17:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:08 DEBUG --- http response --- 17:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:53:08 DEBUG ----- output ----- 17:53:08 DEBUG { 17:53:08 DEBUG "status": "success", 17:53:08 DEBUG "data": { 17:53:08 DEBUG "resultType": "vector", 17:53:08 DEBUG "result": [ 17:53:08 DEBUG { 17:53:08 DEBUG "metric": { 17:53:08 DEBUG "pod": "ds-idrepo-0" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG [...] 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG "pod": "ds-idrepo-2" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 1713545588, 17:53:08 DEBUG "0" 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 INFO [http_cmd]: http status code OK 17:53:08 DEBUG ] 17:53:08 DEBUG --- status code --- 17:53:08 DEBUG } 17:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:08 DEBUG } 17:53:08 DEBUG --- http response --- 17:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:53:08 DEBUG ----- output ----- 17:53:08 DEBUG { 17:53:08 DEBUG "status": "success", 17:53:08 DEBUG "data": { 17:53:08 DEBUG "resultType": "vector", 17:53:08 DEBUG "result": [ 17:53:08 DEBUG { 17:53:08 DEBUG "metric": { 17:53:08 DEBUG "pod": "ds-cts-0" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG [...] 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG "pod": "ds-cts-2" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 1713545588, 17:53:08 DEBUG "450.5333333333333" 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG } 17:53:08 INFO [http_cmd]: http status code OK 17:53:08 DEBUG --- status code --- 17:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:53:08 DEBUG --- http response --- 17:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:53:08 DEBUG ----- output ----- 17:53:08 DEBUG { 17:53:08 DEBUG "status": "success", 17:53:08 DEBUG "data": { 17:53:08 DEBUG "resultType": "vector", 17:53:08 DEBUG "result": [ 17:53:08 DEBUG { 17:53:08 DEBUG "metric": { 17:53:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG [...] 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG 17:53:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:53:08 DEBUG }, 17:53:08 DEBUG "value": [ 17:53:08 DEBUG 1713545588, 17:53:08 DEBUG "1.6234899442217539" 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG ] 17:53:08 DEBUG } 17:53:08 DEBUG } 17:53:09 INFO 17:53:09 INFO [loop_until]: kubectl --namespace=xlou top pods 17:53:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:53:09 INFO [loop_until]: OK (rc = 0) 17:53:09 DEBUG --- stdout --- 17:53:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 2259m 9009Mi am-6fd554f8f7-btd4s 2234m 9031Mi am-6fd554f8f7-wxcsj 2182m 9045Mi ds-cts-0 9705m 13284Mi ds-cts-1 10383m 13157Mi ds-cts-2 20482m 13376Mi ds-idrepo-0 653m 3423Mi ds-idrepo-1 643m 3394Mi ds-idrepo-2 642m 3453Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1216Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1m 18Mi 17:53:09 DEBUG --- stderr --- 17:53:09 DEBUG 17:53:09 INFO 17:53:09 INFO [loop_until]: kubectl --namespace=xlou top node 17:53:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:53:09 INFO [loop_until]: OK (rc = 0) 17:53:09 DEBUG --- stdout --- 17:53:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6182Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 746m 2% 7000Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 51m 0% 2681Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 2394m 8% 12367Mi 11% gke-xlou-cdm-default-pool-1b0769c8-w15q 9645m 32% 16218Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 140m 0% 5250Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2254m 7% 11974Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 107m 0% 3388Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 2437m 8% 11326Mi 10% gke-xlou-cdm-ds-0f601751-j488 732m 2% 6682Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 10648m 35% 17926Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 762m 2% 7233Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 20548m 68% 15614Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 72m 0% 3860Mi 6% 17:53:09 DEBUG --- stderr --- 17:53:09 DEBUG 17:54:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545645" 17:54:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545645" 17:54:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545645" 17:54:05 ERROR [http_cmd]: ERROR 17:54:05 ERROR --- status code --- 17:54:05 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:54:05 ERROR --- http response --- 17:54:05 INFO [http_cmd]: http status code OK 17:54:05 INFO [http_cmd]: http status code OK 17:54:05 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:54:05 DEBUG --- status code --- 17:54:05 DEBUG --- status code --- 17:54:05 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:54:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:05 DEBUG --- http response --- 17:54:05 DEBUG --- http response --- 17:54:05 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:54:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:54:05 DEBUG ----- output ----- 17:54:05 DEBUG { 17:54:05 DEBUG "status": "success", 17:54:05 DEBUG "data": { 17:54:05 DEBUG "resultType": "vector", 17:54:05 DEBUG "result": [ 17:54:05 DEBUG { 17:54:05 DEBUG "metric": { 17:54:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:54:05 DEBUG }, 17:54:05 DEBUG "value": [ 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG [...] 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:54:05 DEBUG }, 17:54:05 DEBUG "value": [ 17:54:05 DEBUG 1713545645, 17:54:05 DEBUG "2329.4333333333334" 17:54:05 DEBUG ] 17:54:05 DEBUG } 17:54:05 DEBUG ] 17:54:05 DEBUG } 17:54:05 DEBUG } 17:54:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545645" 17:54:05 INFO [http_cmd]: http status code OK 17:54:05 DEBUG --- status code --- 17:54:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:05 DEBUG --- http response --- 17:54:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:54:05 DEBUG ----- output ----- 17:54:05 DEBUG { 17:54:05 DEBUG "status": "success", 17:54:05 DEBUG "data": { 17:54:05 DEBUG "resultType": "vector", 17:54:05 DEBUG "result": [ 17:54:05 DEBUG { 17:54:05 DEBUG "metric": { 17:54:05 DEBUG "pod": "ds-idrepo-0" 17:54:05 DEBUG }, 17:54:05 DEBUG "value": [ 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG [...] 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG "pod": "ds-cts-2" 17:54:05 DEBUG }, 17:54:05 DEBUG "value": [ 17:54:05 DEBUG 1713545645, 17:54:05 DEBUG "0" 17:54:05 DEBUG ] 17:54:05 DEBUG } 17:54:05 DEBUG ] 17:54:05 DEBUG } 17:54:05 DEBUG } 17:54:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545645" 17:54:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545645" 17:54:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545645" 17:54:05 INFO [http_cmd]: http status code OK 17:54:05 DEBUG --- status code --- 17:54:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:05 DEBUG --- http response --- 17:54:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:54:05 DEBUG ----- output ----- 17:54:05 DEBUG { 17:54:05 DEBUG "status": "success", 17:54:05 DEBUG "data": { 17:54:05 DEBUG "resultType": "vector", 17:54:05 DEBUG "result": [ 17:54:05 DEBUG { 17:54:05 DEBUG "metric": { 17:54:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:54:05 DEBUG }, 17:54:05 DEBUG "value": [ 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG [...] 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:54:05 DEBUG }, 17:54:05 DEBUG "value": [ 17:54:05 DEBUG 1713545645, 17:54:05 DEBUG "0" 17:54:05 DEBUG ] 17:54:05 DEBUG } 17:54:05 DEBUG ] 17:54:05 DEBUG } 17:54:05 DEBUG } 17:54:05 INFO [http_cmd]: http status code OK 17:54:05 INFO [http_cmd]: http status code OK 17:54:05 DEBUG --- status code --- 17:54:05 DEBUG --- status code --- 17:54:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:05 DEBUG --- http response --- 17:54:05 DEBUG --- http response --- 17:54:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:54:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:54:05 DEBUG ----- output ----- 17:54:05 DEBUG ----- output ----- 17:54:05 DEBUG { 17:54:05 DEBUG { 17:54:05 DEBUG "status": "success", 17:54:05 DEBUG "status": "success", 17:54:05 DEBUG "data": { 17:54:05 DEBUG "data": { 17:54:05 DEBUG "resultType": "vector", 17:54:05 DEBUG "resultType": "vector", 17:54:05 DEBUG "result": [ 17:54:05 DEBUG "result": [ 17:54:05 DEBUG { 17:54:05 DEBUG { 17:54:05 DEBUG "metric": { 17:54:05 DEBUG "metric": { 17:54:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:54:05 DEBUG "pod": "ds-idrepo-1" 17:54:05 DEBUG }, 17:54:05 DEBUG }, 17:54:05 DEBUG "value": [ 17:54:05 DEBUG "value": [ 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG [...] 17:54:05 DEBUG [...] 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:54:05 DEBUG "pod": "idm-b85f9d978-txpwv" 17:54:05 DEBUG }, 17:54:05 DEBUG }, 17:54:05 DEBUG "value": [ 17:54:05 DEBUG "value": [ 17:54:05 DEBUG 1713545645, 17:54:05 DEBUG 1713545645, 17:54:05 DEBUG "0" 17:54:05 DEBUG "0" 17:54:05 DEBUG ] 17:54:05 DEBUG ] 17:54:05 DEBUG } 17:54:05 DEBUG } 17:54:05 DEBUG ] 17:54:05 DEBUG ] 17:54:05 DEBUG } 17:54:05 DEBUG } 17:54:05 DEBUG } 17:54:05 DEBUG } 17:54:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545645" 17:54:05 INFO [http_cmd]: http status code OK 17:54:05 DEBUG --- status code --- 17:54:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:05 DEBUG --- http response --- 17:54:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:54:05 DEBUG ----- output ----- 17:54:05 DEBUG { 17:54:05 DEBUG "status": "success", 17:54:05 DEBUG "data": { 17:54:05 DEBUG "resultType": "vector", 17:54:05 DEBUG "result": [ 17:54:05 DEBUG { 17:54:05 DEBUG "metric": { 17:54:05 DEBUG "instance": "10.142.0.107:9100" 17:54:05 DEBUG }, 17:54:05 DEBUG "value": [ 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG [...] 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG 17:54:05 DEBUG "instance": "10.142.0.43:9100" 17:54:05 DEBUG }, 17:54:05 DEBUG "value": [ 17:54:05 DEBUG 1713545645, 17:54:05 DEBUG "0.6396666666666628" 17:54:05 DEBUG ] 17:54:05 DEBUG } 17:54:05 DEBUG ] 17:54:05 DEBUG } 17:54:05 DEBUG } 17:54:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545646" 17:54:06 INFO [http_cmd]: http status code OK 17:54:06 DEBUG --- status code --- 17:54:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:06 DEBUG --- http response --- 17:54:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:54:06 DEBUG ----- output ----- 17:54:06 DEBUG { 17:54:06 DEBUG "status": "success", 17:54:06 DEBUG "data": { 17:54:06 DEBUG "resultType": "vector", 17:54:06 DEBUG "result": [ 17:54:06 DEBUG { 17:54:06 DEBUG "metric": { 17:54:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:54:06 DEBUG }, 17:54:06 DEBUG "value": [ 17:54:06 DEBUG 17:54:06 DEBUG 17:54:06 DEBUG 17:54:06 DEBUG [...] 17:54:06 DEBUG 17:54:06 DEBUG 17:54:06 DEBUG 17:54:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:54:06 DEBUG }, 17:54:06 DEBUG "value": [ 17:54:06 DEBUG 1713545646, 17:54:06 DEBUG "3842723.5" 17:54:06 DEBUG ] 17:54:06 DEBUG } 17:54:06 DEBUG ] 17:54:06 DEBUG } 17:54:06 DEBUG } 17:54:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545646" 17:54:06 INFO [http_cmd]: http status code OK 17:54:06 DEBUG --- status code --- 17:54:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:06 DEBUG --- http response --- 17:54:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:54:06 DEBUG ----- output ----- 17:54:06 DEBUG { 17:54:06 DEBUG "status": "success", 17:54:06 DEBUG "data": { 17:54:06 DEBUG "resultType": "vector", 17:54:06 DEBUG "result": [ 17:54:06 DEBUG { 17:54:06 DEBUG "metric": { 17:54:06 DEBUG "instance": "10.142.0.107:9100" 17:54:06 DEBUG }, 17:54:06 DEBUG "value": [ 17:54:06 DEBUG 17:54:06 DEBUG 17:54:06 DEBUG 17:54:06 DEBUG [...] 17:54:06 DEBUG 17:54:06 DEBUG 17:54:06 DEBUG 17:54:06 DEBUG "instance": "10.142.0.43:9100" 17:54:06 DEBUG }, 17:54:06 DEBUG "value": [ 17:54:06 DEBUG 1713545646, 17:54:06 DEBUG "1798194596864" 17:54:06 DEBUG ] 17:54:06 DEBUG } 17:54:06 DEBUG ] 17:54:06 DEBUG } 17:54:06 DEBUG } 17:54:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545647" 17:54:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545647" 17:54:07 INFO [http_cmd]: http status code OK 17:54:07 DEBUG --- status code --- 17:54:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:07 DEBUG --- http response --- 17:54:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:54:07 DEBUG ----- output ----- 17:54:07 DEBUG { 17:54:07 DEBUG "status": "success", 17:54:07 DEBUG "data": { 17:54:07 DEBUG "resultType": "vector", 17:54:07 DEBUG "result": [ 17:54:07 DEBUG { 17:54:07 DEBUG "metric": { 17:54:07 DEBUG "pod": "ds-idrepo-0" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG [...] 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG "pod": "ds-idrepo-2" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 1713545647, 17:54:07 DEBUG "0" 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG } 17:54:07 INFO [http_cmd]: http status code OK 17:54:07 DEBUG --- status code --- 17:54:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:07 DEBUG --- http response --- 17:54:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:54:07 DEBUG ----- output ----- 17:54:07 DEBUG { 17:54:07 DEBUG "status": "success", 17:54:07 DEBUG "data": { 17:54:07 DEBUG "resultType": "vector", 17:54:07 DEBUG "result": [ 17:54:07 DEBUG { 17:54:07 DEBUG "metric": { 17:54:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG [...] 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 1713545647, 17:54:07 DEBUG "491790336" 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG } 17:54:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545647" 17:54:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545647" 17:54:07 INFO [http_cmd]: http status code OK 17:54:07 DEBUG --- status code --- 17:54:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:07 DEBUG --- http response --- 17:54:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:54:07 DEBUG ----- output ----- 17:54:07 DEBUG { 17:54:07 DEBUG "status": "success", 17:54:07 DEBUG "data": { 17:54:07 DEBUG "resultType": "vector", 17:54:07 DEBUG "result": [ 17:54:07 DEBUG { 17:54:07 DEBUG "metric": { 17:54:07 DEBUG "pod": "ds-cts-0" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG [...] 17:54:07 INFO [http_cmd]: http status code OK 17:54:07 DEBUG 17:54:07 DEBUG --- status code --- 17:54:07 DEBUG 17:54:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:07 DEBUG 17:54:07 DEBUG --- http response --- 17:54:07 DEBUG "pod": "ds-cts-2" 17:54:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:54:07 DEBUG }, 17:54:07 DEBUG ----- output ----- 17:54:07 DEBUG "value": [ 17:54:07 DEBUG { 17:54:07 DEBUG 1713545647, 17:54:07 DEBUG "status": "success", 17:54:07 DEBUG "0.00030000000000001136" 17:54:07 DEBUG "data": { 17:54:07 DEBUG ] 17:54:07 DEBUG "resultType": "vector", 17:54:07 DEBUG } 17:54:07 DEBUG "result": [ 17:54:07 DEBUG ] 17:54:07 DEBUG { 17:54:07 DEBUG } 17:54:07 DEBUG "metric": { 17:54:07 DEBUG } 17:54:07 DEBUG "pod": "ds-cts-0" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG [...] 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG "pod": "ds-cts-2" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 1713545647, 17:54:07 DEBUG "34.766666666666666" 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG } 17:54:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545647" 17:54:07 INFO [http_cmd]: http status code OK 17:54:07 DEBUG --- status code --- 17:54:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:07 DEBUG --- http response --- 17:54:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:54:07 DEBUG ----- output ----- 17:54:07 DEBUG { 17:54:07 DEBUG "status": "success", 17:54:07 DEBUG "data": { 17:54:07 DEBUG "resultType": "vector", 17:54:07 DEBUG "result": [ 17:54:07 DEBUG { 17:54:07 DEBUG "metric": { 17:54:07 DEBUG "pod": "ds-idrepo-0" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG [...] 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG "pod": "ds-idrepo-2" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 1713545647, 17:54:07 DEBUG "0" 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG } 17:54:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545647" 17:54:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545647" 17:54:07 INFO [http_cmd]: http status code OK 17:54:07 INFO [http_cmd]: http status code OK 17:54:07 DEBUG --- status code --- 17:54:07 DEBUG --- status code --- 17:54:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:07 DEBUG --- http response --- 17:54:07 DEBUG --- http response --- 17:54:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:54:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:54:07 DEBUG ----- output ----- 17:54:07 DEBUG ----- output ----- 17:54:07 DEBUG { 17:54:07 DEBUG { 17:54:07 DEBUG "status": "success", 17:54:07 DEBUG "status": "success", 17:54:07 DEBUG "data": { 17:54:07 DEBUG "data": { 17:54:07 DEBUG "resultType": "vector", 17:54:07 DEBUG "resultType": "vector", 17:54:07 DEBUG "result": [ 17:54:07 DEBUG "result": [ 17:54:07 DEBUG { 17:54:07 DEBUG { 17:54:07 DEBUG "metric": { 17:54:07 DEBUG "metric": { 17:54:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:54:07 DEBUG "instance": "10.142.0.107:9100" 17:54:07 DEBUG }, 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG [...] 17:54:07 DEBUG [...] 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:54:07 DEBUG "instance": "10.142.0.43:9100" 17:54:07 DEBUG }, 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 1713545647, 17:54:07 DEBUG 1713545647, 17:54:07 DEBUG "0" 17:54:07 DEBUG "16210029.466666667" 17:54:07 DEBUG ] 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG } 17:54:07 DEBUG ] 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG } 17:54:07 DEBUG } 17:54:07 DEBUG } 17:54:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545647" 17:54:07 INFO [http_cmd]: http status code OK 17:54:07 DEBUG --- status code --- 17:54:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:07 DEBUG --- http response --- 17:54:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:54:07 DEBUG ----- output ----- 17:54:07 DEBUG { 17:54:07 DEBUG "status": "success", 17:54:07 DEBUG "data": { 17:54:07 DEBUG "resultType": "vector", 17:54:07 DEBUG "result": [ 17:54:07 DEBUG { 17:54:07 DEBUG "metric": { 17:54:07 DEBUG "instance": "10.142.0.107:9100" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG [...] 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG "instance": "10.142.0.43:9100" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 1713545647, 17:54:07 DEBUG "16201304.6" 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG } 17:54:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545647" 17:54:07 INFO [http_cmd]: http status code OK 17:54:07 DEBUG --- status code --- 17:54:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:07 DEBUG --- http response --- 17:54:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:54:07 DEBUG ----- output ----- 17:54:07 DEBUG { 17:54:07 DEBUG "status": "success", 17:54:07 DEBUG "data": { 17:54:07 DEBUG "resultType": "vector", 17:54:07 DEBUG "result": [ 17:54:07 DEBUG { 17:54:07 DEBUG "metric": { 17:54:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG [...] 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG 17:54:07 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:54:07 DEBUG }, 17:54:07 DEBUG "value": [ 17:54:07 DEBUG 1713545647, 17:54:07 DEBUG "0" 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG ] 17:54:07 DEBUG } 17:54:07 DEBUG } 17:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545648" 17:54:08 INFO [http_cmd]: http status code OK 17:54:08 DEBUG --- status code --- 17:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:08 DEBUG --- http response --- 17:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:54:08 DEBUG ----- output ----- 17:54:08 DEBUG { 17:54:08 DEBUG "status": "success", 17:54:08 DEBUG "data": { 17:54:08 DEBUG "resultType": "vector", 17:54:08 DEBUG "result": [ 17:54:08 DEBUG { 17:54:08 DEBUG "metric": { 17:54:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:54:08 DEBUG }, 17:54:08 DEBUG "value": [ 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG [...] 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:54:08 DEBUG }, 17:54:08 DEBUG "value": [ 17:54:08 DEBUG 1713545648, 17:54:08 DEBUG "0" 17:54:08 DEBUG ] 17:54:08 DEBUG } 17:54:08 DEBUG ] 17:54:08 DEBUG } 17:54:08 DEBUG } 17:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545648" 17:54:08 INFO [http_cmd]: http status code OK 17:54:08 DEBUG --- status code --- 17:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:08 DEBUG --- http response --- 17:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:54:08 DEBUG ----- output ----- 17:54:08 DEBUG { 17:54:08 DEBUG "status": "success", 17:54:08 DEBUG "data": { 17:54:08 DEBUG "resultType": "vector", 17:54:08 DEBUG "result": [ 17:54:08 DEBUG { 17:54:08 DEBUG "metric": { 17:54:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:54:08 DEBUG }, 17:54:08 DEBUG "value": [ 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG [...] 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:54:08 DEBUG }, 17:54:08 DEBUG "value": [ 17:54:08 DEBUG 1713545648, 17:54:08 DEBUG "0.025500000000000002" 17:54:08 DEBUG ] 17:54:08 DEBUG } 17:54:08 DEBUG ] 17:54:08 DEBUG } 17:54:08 DEBUG } 17:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545648" 17:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545648" 17:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545648" 17:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545648" 17:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545648" 17:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545648" 17:54:08 INFO [http_cmd]: http status code OK 17:54:08 DEBUG --- status code --- 17:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:08 DEBUG --- http response --- 17:54:08 INFO [http_cmd]: http status code OK 17:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:54:08 INFO [http_cmd]: http status code OK 17:54:08 DEBUG --- status code --- 17:54:08 DEBUG ----- output ----- 17:54:08 DEBUG --- status code --- 17:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:08 DEBUG { 17:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:08 DEBUG --- http response --- 17:54:08 DEBUG "status": "success", 17:54:08 DEBUG --- http response --- 17:54:08 DEBUG "data": { 17:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:54:08 INFO [http_cmd]: http status code OK 17:54:08 DEBUG "resultType": "vector", 17:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:54:08 DEBUG ----- output ----- 17:54:08 INFO [http_cmd]: http status code OK 17:54:08 DEBUG --- status code --- 17:54:08 DEBUG "result": [ 17:54:08 DEBUG ----- output ----- 17:54:08 DEBUG { 17:54:08 DEBUG --- status code --- 17:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:08 DEBUG { 17:54:08 DEBUG { 17:54:08 DEBUG "status": "success", 17:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:08 DEBUG --- http response --- 17:54:08 DEBUG "metric": { 17:54:08 DEBUG "status": "success", 17:54:08 DEBUG "data": { 17:54:08 DEBUG --- http response --- 17:54:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:54:08 DEBUG "data": { 17:54:08 DEBUG "resultType": "vector", 17:54:08 DEBUG }, 17:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:54:08 DEBUG ----- output ----- 17:54:08 DEBUG "resultType": "vector", 17:54:08 DEBUG "result": [ 17:54:08 DEBUG "value": [ 17:54:08 DEBUG ----- output ----- 17:54:08 DEBUG { 17:54:08 DEBUG "result": [ 17:54:08 DEBUG { 17:54:08 DEBUG 17:54:08 DEBUG { 17:54:08 DEBUG "status": "success", 17:54:08 DEBUG { 17:54:08 DEBUG "metric": { 17:54:08 DEBUG 17:54:08 DEBUG "status": "success", 17:54:08 DEBUG "data": { 17:54:08 DEBUG "metric": { 17:54:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:54:08 DEBUG 17:54:08 DEBUG "data": { 17:54:08 DEBUG "resultType": "vector", 17:54:08 DEBUG "instance": "10.142.0.107:9100" 17:54:08 DEBUG }, 17:54:08 DEBUG [...] 17:54:08 DEBUG "resultType": "vector", 17:54:08 DEBUG "result": [ 17:54:08 DEBUG }, 17:54:08 DEBUG "value": [ 17:54:08 DEBUG 17:54:08 DEBUG "result": [ 17:54:08 DEBUG { 17:54:08 DEBUG "value": [ 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG { 17:54:08 DEBUG "metric": { 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG "metric": { 17:54:08 DEBUG "pod": "ds-idrepo-0" 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:54:08 DEBUG "pod": "ds-cts-0" 17:54:08 DEBUG }, 17:54:08 DEBUG 17:54:08 DEBUG [...] 17:54:08 DEBUG }, 17:54:08 DEBUG }, 17:54:08 DEBUG "value": [ 17:54:08 DEBUG [...] 17:54:08 DEBUG 17:54:08 INFO [http_cmd]: http status code OK 17:54:08 DEBUG "value": [ 17:54:08 DEBUG "value": [ 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG --- status code --- 17:54:08 DEBUG 1713545648, 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:54:08 DEBUG "0" 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:54:08 DEBUG --- http response --- 17:54:08 DEBUG ] 17:54:08 DEBUG 17:54:08 DEBUG [...] 17:54:08 DEBUG "instance": "10.142.0.43:9100" 17:54:08 DEBUG }, 17:54:08 DEBUG } 17:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:54:08 DEBUG [...] 17:54:08 DEBUG 17:54:08 DEBUG }, 17:54:08 DEBUG "value": [ 17:54:08 DEBUG ] 17:54:08 DEBUG ----- output ----- 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG "value": [ 17:54:08 DEBUG 1713545648, 17:54:08 DEBUG } 17:54:08 DEBUG { 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG 1713545648, 17:54:08 DEBUG "0.00037275535411552266" 17:54:08 DEBUG } 17:54:08 DEBUG "status": "success", 17:54:08 DEBUG 17:54:08 DEBUG "pod": "ds-idrepo-2" 17:54:08 DEBUG "0.6629000000000066" 17:54:08 DEBUG ] 17:54:08 DEBUG "data": { 17:54:08 DEBUG "pod": "ds-cts-2" 17:54:08 DEBUG }, 17:54:08 DEBUG ] 17:54:08 DEBUG } 17:54:08 DEBUG "resultType": "vector", 17:54:08 DEBUG }, 17:54:08 DEBUG "value": [ 17:54:08 DEBUG } 17:54:08 DEBUG ] 17:54:08 DEBUG "result": [ 17:54:08 DEBUG "value": [ 17:54:08 DEBUG 1713545648, 17:54:08 DEBUG ] 17:54:08 DEBUG } 17:54:08 DEBUG { 17:54:08 DEBUG 1713545648, 17:54:08 DEBUG "0" 17:54:08 DEBUG } 17:54:08 DEBUG } 17:54:08 DEBUG "metric": { 17:54:08 DEBUG "586.1666666666666" 17:54:08 DEBUG ] 17:54:08 DEBUG } 17:54:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:54:08 DEBUG ] 17:54:08 DEBUG } 17:54:08 DEBUG }, 17:54:08 DEBUG } 17:54:08 DEBUG ] 17:54:08 DEBUG "value": [ 17:54:08 DEBUG ] 17:54:08 DEBUG } 17:54:08 DEBUG 17:54:08 DEBUG } 17:54:08 DEBUG } 17:54:08 DEBUG 17:54:08 DEBUG } 17:54:08 DEBUG 17:54:08 DEBUG [...] 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG 17:54:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:54:08 DEBUG }, 17:54:08 DEBUG "value": [ 17:54:08 DEBUG 1713545648, 17:54:08 DEBUG "1.2304221288473915" 17:54:08 DEBUG ] 17:54:08 DEBUG } 17:54:08 DEBUG ] 17:54:08 DEBUG } 17:54:08 DEBUG } 17:54:09 INFO 17:54:09 INFO [loop_until]: kubectl --namespace=xlou top pods 17:54:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:54:09 INFO [loop_until]: OK (rc = 0) 17:54:09 DEBUG --- stdout --- 17:54:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 2211m 8910Mi am-6fd554f8f7-btd4s 2082m 8905Mi am-6fd554f8f7-wxcsj 2140m 8923Mi ds-cts-0 9215m 13281Mi ds-cts-1 9305m 13366Mi ds-cts-2 12333m 12921Mi ds-idrepo-0 1401m 3525Mi ds-idrepo-1 1111m 3404Mi ds-idrepo-2 1071m 3465Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1063m 450Mi 17:54:09 DEBUG --- stderr --- 17:54:09 DEBUG 17:54:09 INFO 17:54:09 INFO [loop_until]: kubectl --namespace=xlou top node 17:54:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:54:09 INFO [loop_until]: OK (rc = 0) 17:54:09 DEBUG --- stdout --- 17:54:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6183Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 1482m 4% 7002Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 49m 0% 2680Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 2179m 7% 12228Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 8919m 29% 16772Mi 15% gke-xlou-cdm-default-pool-1b0769c8-wpwl 165m 0% 5209Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2167m 7% 11831Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 77m 0% 3283Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2314m 7% 11176Mi 10% gke-xlou-cdm-ds-0f601751-j488 1138m 3% 6656Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9154m 30% 17884Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 1101m 3% 7221Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 11030m 36% 15371Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 1090m 6% 4325Mi 7% 17:54:09 DEBUG --- stderr --- 17:54:09 DEBUG 17:55:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545705" 17:55:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545705" 17:55:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545705" 17:55:05 INFO [http_cmd]: http status code OK 17:55:05 ERROR [http_cmd]: ERROR 17:55:05 INFO [http_cmd]: http status code OK 17:55:05 DEBUG --- status code --- 17:55:05 ERROR --- status code --- 17:55:05 DEBUG --- status code --- 17:55:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:05 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:55:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:05 DEBUG --- http response --- 17:55:05 ERROR --- http response --- 17:55:05 DEBUG --- http response --- 17:55:05 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:55:05 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:55:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:55:05 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:55:05 DEBUG ----- output ----- 17:55:05 DEBUG { 17:55:05 DEBUG "status": "success", 17:55:05 DEBUG "data": { 17:55:05 DEBUG "resultType": "vector", 17:55:05 DEBUG "result": [ 17:55:05 DEBUG { 17:55:05 DEBUG "metric": { 17:55:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:55:05 DEBUG }, 17:55:05 DEBUG "value": [ 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG [...] 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:55:05 DEBUG }, 17:55:05 DEBUG "value": [ 17:55:05 DEBUG 1713545705, 17:55:05 DEBUG "2331.8333333333335" 17:55:05 DEBUG ] 17:55:05 DEBUG } 17:55:05 DEBUG ] 17:55:05 DEBUG } 17:55:05 DEBUG } 17:55:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545705" 17:55:05 INFO [http_cmd]: http status code OK 17:55:05 DEBUG --- status code --- 17:55:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:05 DEBUG --- http response --- 17:55:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:55:05 DEBUG ----- output ----- 17:55:05 DEBUG { 17:55:05 DEBUG "status": "success", 17:55:05 DEBUG "data": { 17:55:05 DEBUG "resultType": "vector", 17:55:05 DEBUG "result": [ 17:55:05 DEBUG { 17:55:05 DEBUG "metric": { 17:55:05 DEBUG "pod": "ds-idrepo-0" 17:55:05 DEBUG }, 17:55:05 DEBUG "value": [ 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG [...] 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG "pod": "ds-cts-2" 17:55:05 DEBUG }, 17:55:05 DEBUG "value": [ 17:55:05 DEBUG 1713545705, 17:55:05 DEBUG "0" 17:55:05 DEBUG ] 17:55:05 DEBUG } 17:55:05 DEBUG ] 17:55:05 DEBUG } 17:55:05 DEBUG } 17:55:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545705" 17:55:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545705" 17:55:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545705" 17:55:05 INFO [http_cmd]: http status code OK 17:55:05 DEBUG --- status code --- 17:55:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:05 DEBUG --- http response --- 17:55:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 17:55:05 DEBUG ----- output ----- 17:55:05 DEBUG { 17:55:05 DEBUG "status": "success", 17:55:05 DEBUG "data": { 17:55:05 INFO [http_cmd]: http status code OK 17:55:05 INFO [http_cmd]: http status code OK 17:55:05 DEBUG "resultType": "vector", 17:55:05 DEBUG --- status code --- 17:55:05 DEBUG --- status code --- 17:55:05 DEBUG "result": [ 17:55:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:05 DEBUG { 17:55:05 DEBUG --- http response --- 17:55:05 DEBUG --- http response --- 17:55:05 DEBUG "metric": { 17:55:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:55:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:55:05 DEBUG "pod": "ds-idrepo-0" 17:55:05 DEBUG ----- output ----- 17:55:05 DEBUG ----- output ----- 17:55:05 DEBUG }, 17:55:05 DEBUG { 17:55:05 DEBUG { 17:55:05 DEBUG "value": [ 17:55:05 DEBUG "status": "success", 17:55:05 DEBUG "status": "success", 17:55:05 DEBUG 17:55:05 DEBUG "data": { 17:55:05 DEBUG "data": { 17:55:05 DEBUG 17:55:05 DEBUG "resultType": "vector", 17:55:05 DEBUG "resultType": "vector", 17:55:05 DEBUG 17:55:05 DEBUG "result": [ 17:55:05 DEBUG "result": [ 17:55:05 DEBUG [...] 17:55:05 DEBUG { 17:55:05 DEBUG { 17:55:05 DEBUG 17:55:05 DEBUG "metric": { 17:55:05 DEBUG "metric": { 17:55:05 DEBUG 17:55:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:55:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:55:05 DEBUG 17:55:05 DEBUG }, 17:55:05 DEBUG }, 17:55:05 DEBUG "pod": "ds-idrepo-2" 17:55:05 DEBUG "value": [ 17:55:05 DEBUG "value": [ 17:55:05 DEBUG }, 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG "value": [ 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG 1713545705, 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG "0" 17:55:05 DEBUG [...] 17:55:05 DEBUG [...] 17:55:05 DEBUG ] 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG } 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG ] 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG } 17:55:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:55:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:55:05 DEBUG } 17:55:05 DEBUG }, 17:55:05 DEBUG }, 17:55:05 DEBUG "value": [ 17:55:05 DEBUG "value": [ 17:55:05 DEBUG 1713545705, 17:55:05 DEBUG 1713545705, 17:55:05 DEBUG "0" 17:55:05 DEBUG "0" 17:55:05 DEBUG ] 17:55:05 DEBUG ] 17:55:05 DEBUG } 17:55:05 DEBUG } 17:55:05 DEBUG ] 17:55:05 DEBUG ] 17:55:05 DEBUG } 17:55:05 DEBUG } 17:55:05 DEBUG } 17:55:05 DEBUG } 17:55:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545705" 17:55:05 INFO [http_cmd]: http status code OK 17:55:05 DEBUG --- status code --- 17:55:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:05 DEBUG --- http response --- 17:55:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:55:05 DEBUG ----- output ----- 17:55:05 DEBUG { 17:55:05 DEBUG "status": "success", 17:55:05 DEBUG "data": { 17:55:05 DEBUG "resultType": "vector", 17:55:05 DEBUG "result": [ 17:55:05 DEBUG { 17:55:05 DEBUG "metric": { 17:55:05 DEBUG "instance": "10.142.0.107:9100" 17:55:05 DEBUG }, 17:55:05 DEBUG "value": [ 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG [...] 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG 17:55:05 DEBUG "instance": "10.142.0.43:9100" 17:55:05 DEBUG }, 17:55:05 DEBUG "value": [ 17:55:05 DEBUG 1713545705, 17:55:05 DEBUG "0.24366666666666867" 17:55:05 DEBUG ] 17:55:05 DEBUG } 17:55:05 DEBUG ] 17:55:05 DEBUG } 17:55:05 DEBUG } 17:55:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545706" 17:55:06 INFO [http_cmd]: http status code OK 17:55:06 DEBUG --- status code --- 17:55:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:06 DEBUG --- http response --- 17:55:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:55:06 DEBUG ----- output ----- 17:55:06 DEBUG { 17:55:06 DEBUG "status": "success", 17:55:06 DEBUG "data": { 17:55:06 DEBUG "resultType": "vector", 17:55:06 DEBUG "result": [ 17:55:06 DEBUG { 17:55:06 DEBUG "metric": { 17:55:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:55:06 DEBUG }, 17:55:06 DEBUG "value": [ 17:55:06 DEBUG 17:55:06 DEBUG 17:55:06 DEBUG 17:55:06 DEBUG [...] 17:55:06 DEBUG 17:55:06 DEBUG 17:55:06 DEBUG 17:55:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:55:06 DEBUG }, 17:55:06 DEBUG "value": [ 17:55:06 DEBUG 1713545706, 17:55:06 DEBUG "3703007.5" 17:55:06 DEBUG ] 17:55:06 DEBUG } 17:55:06 DEBUG ] 17:55:06 DEBUG } 17:55:06 DEBUG } 17:55:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545706" 17:55:06 INFO [http_cmd]: http status code OK 17:55:06 DEBUG --- status code --- 17:55:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:06 DEBUG --- http response --- 17:55:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:55:06 DEBUG ----- output ----- 17:55:06 DEBUG { 17:55:06 DEBUG "status": "success", 17:55:06 DEBUG "data": { 17:55:06 DEBUG "resultType": "vector", 17:55:06 DEBUG "result": [ 17:55:06 DEBUG { 17:55:06 DEBUG "metric": { 17:55:06 DEBUG "instance": "10.142.0.107:9100" 17:55:06 DEBUG }, 17:55:06 DEBUG "value": [ 17:55:06 DEBUG 17:55:06 DEBUG 17:55:06 DEBUG 17:55:06 DEBUG [...] 17:55:06 DEBUG 17:55:06 DEBUG 17:55:06 DEBUG 17:55:06 DEBUG "instance": "10.142.0.43:9100" 17:55:06 DEBUG }, 17:55:06 DEBUG "value": [ 17:55:06 DEBUG 1713545706, 17:55:06 DEBUG "1799682894848" 17:55:06 DEBUG ] 17:55:06 DEBUG } 17:55:06 DEBUG ] 17:55:06 DEBUG } 17:55:06 DEBUG } 17:55:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545707" 17:55:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545707" 17:55:07 INFO [http_cmd]: http status code OK 17:55:07 DEBUG --- status code --- 17:55:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:07 DEBUG --- http response --- 17:55:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:55:07 DEBUG ----- output ----- 17:55:07 DEBUG { 17:55:07 DEBUG "status": "success", 17:55:07 DEBUG "data": { 17:55:07 DEBUG "resultType": "vector", 17:55:07 DEBUG "result": [ 17:55:07 DEBUG { 17:55:07 DEBUG "metric": { 17:55:07 DEBUG "pod": "ds-idrepo-0" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG [...] 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG "pod": "ds-idrepo-2" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 1713545707, 17:55:07 DEBUG "0" 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG } 17:55:07 INFO [http_cmd]: http status code OK 17:55:07 DEBUG --- status code --- 17:55:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:07 DEBUG --- http response --- 17:55:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:55:07 DEBUG ----- output ----- 17:55:07 DEBUG { 17:55:07 DEBUG "status": "success", 17:55:07 DEBUG "data": { 17:55:07 DEBUG "resultType": "vector", 17:55:07 DEBUG "result": [ 17:55:07 DEBUG { 17:55:07 DEBUG "metric": { 17:55:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG [...] 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545707" 17:55:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545707" 17:55:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 1713545707, 17:55:07 DEBUG "804368384" 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG } 17:55:07 INFO [http_cmd]: http status code OK 17:55:07 DEBUG --- status code --- 17:55:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:07 DEBUG --- http response --- 17:55:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:55:07 DEBUG ----- output ----- 17:55:07 DEBUG { 17:55:07 DEBUG "status": "success", 17:55:07 DEBUG "data": { 17:55:07 DEBUG "resultType": "vector", 17:55:07 DEBUG "result": [ 17:55:07 DEBUG { 17:55:07 DEBUG "metric": { 17:55:07 DEBUG "pod": "ds-cts-0" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG [...] 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG "pod": "ds-cts-2" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 1713545707, 17:55:07 DEBUG "8.966666666666667" 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG } 17:55:07 INFO [http_cmd]: http status code OK 17:55:07 DEBUG --- status code --- 17:55:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:07 DEBUG --- http response --- 17:55:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:55:07 DEBUG ----- output ----- 17:55:07 DEBUG { 17:55:07 DEBUG "status": "success", 17:55:07 DEBUG "data": { 17:55:07 DEBUG "resultType": "vector", 17:55:07 DEBUG "result": [ 17:55:07 DEBUG { 17:55:07 DEBUG "metric": { 17:55:07 DEBUG "pod": "ds-cts-0" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG [...] 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG "pod": "ds-cts-2" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 1713545707, 17:55:07 DEBUG "0.0001666666666666667" 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG } 17:55:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545707" 17:55:07 INFO [http_cmd]: http status code OK 17:55:07 DEBUG --- status code --- 17:55:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:07 DEBUG --- http response --- 17:55:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:55:07 DEBUG ----- output ----- 17:55:07 DEBUG { 17:55:07 DEBUG "status": "success", 17:55:07 DEBUG "data": { 17:55:07 DEBUG "resultType": "vector", 17:55:07 DEBUG "result": [ 17:55:07 DEBUG { 17:55:07 DEBUG "metric": { 17:55:07 DEBUG "pod": "ds-idrepo-0" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG [...] 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG "pod": "ds-idrepo-2" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 1713545707, 17:55:07 DEBUG "0" 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG } 17:55:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545707" 17:55:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545707" 17:55:07 INFO [http_cmd]: http status code OK 17:55:07 DEBUG --- status code --- 17:55:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:07 DEBUG --- http response --- 17:55:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:55:07 DEBUG ----- output ----- 17:55:07 DEBUG { 17:55:07 DEBUG "status": "success", 17:55:07 DEBUG "data": { 17:55:07 DEBUG "resultType": "vector", 17:55:07 DEBUG "result": [ 17:55:07 DEBUG { 17:55:07 DEBUG "metric": { 17:55:07 DEBUG "instance": "10.142.0.107:9100" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG [...] 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG "instance": "10.142.0.43:9100" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 1713545707, 17:55:07 DEBUG "16187196.766666666" 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG } 17:55:07 INFO [http_cmd]: http status code OK 17:55:07 DEBUG --- status code --- 17:55:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:07 DEBUG --- http response --- 17:55:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:55:07 DEBUG ----- output ----- 17:55:07 DEBUG { 17:55:07 DEBUG "status": "success", 17:55:07 DEBUG "data": { 17:55:07 DEBUG "resultType": "vector", 17:55:07 DEBUG "result": [ 17:55:07 DEBUG { 17:55:07 DEBUG "metric": { 17:55:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG [...] 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 1713545707, 17:55:07 DEBUG "0" 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG } 17:55:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545707" 17:55:07 INFO [http_cmd]: http status code OK 17:55:07 DEBUG --- status code --- 17:55:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:07 DEBUG --- http response --- 17:55:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:55:07 DEBUG ----- output ----- 17:55:07 DEBUG { 17:55:07 DEBUG "status": "success", 17:55:07 DEBUG "data": { 17:55:07 DEBUG "resultType": "vector", 17:55:07 DEBUG "result": [ 17:55:07 DEBUG { 17:55:07 DEBUG "metric": { 17:55:07 DEBUG "instance": "10.142.0.107:9100" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG [...] 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG "instance": "10.142.0.43:9100" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 1713545707, 17:55:07 DEBUG "16178103.8" 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG } 17:55:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545707" 17:55:07 INFO [http_cmd]: http status code OK 17:55:07 DEBUG --- status code --- 17:55:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:07 DEBUG --- http response --- 17:55:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:55:07 DEBUG ----- output ----- 17:55:07 DEBUG { 17:55:07 DEBUG "status": "success", 17:55:07 DEBUG "data": { 17:55:07 DEBUG "resultType": "vector", 17:55:07 DEBUG "result": [ 17:55:07 DEBUG { 17:55:07 DEBUG "metric": { 17:55:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG [...] 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG 17:55:07 DEBUG "pod": "idm-b85f9d978-wnhxb" 17:55:07 DEBUG }, 17:55:07 DEBUG "value": [ 17:55:07 DEBUG 1713545707, 17:55:07 DEBUG "0" 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG ] 17:55:07 DEBUG } 17:55:07 DEBUG } 17:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545708" 17:55:08 INFO [http_cmd]: http status code OK 17:55:08 DEBUG --- status code --- 17:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:08 DEBUG --- http response --- 17:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:55:08 DEBUG ----- output ----- 17:55:08 DEBUG { 17:55:08 DEBUG "status": "success", 17:55:08 DEBUG "data": { 17:55:08 DEBUG "resultType": "vector", 17:55:08 DEBUG "result": [ 17:55:08 DEBUG { 17:55:08 DEBUG "metric": { 17:55:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:55:08 DEBUG }, 17:55:08 DEBUG "value": [ 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG [...] 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:55:08 DEBUG }, 17:55:08 DEBUG "value": [ 17:55:08 DEBUG 1713545708, 17:55:08 DEBUG "0" 17:55:08 DEBUG ] 17:55:08 DEBUG } 17:55:08 DEBUG ] 17:55:08 DEBUG } 17:55:08 DEBUG } 17:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545708" 17:55:08 INFO [http_cmd]: http status code OK 17:55:08 DEBUG --- status code --- 17:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:08 DEBUG --- http response --- 17:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:55:08 DEBUG ----- output ----- 17:55:08 DEBUG { 17:55:08 DEBUG "status": "success", 17:55:08 DEBUG "data": { 17:55:08 DEBUG "resultType": "vector", 17:55:08 DEBUG "result": [ 17:55:08 DEBUG { 17:55:08 DEBUG "metric": { 17:55:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:55:08 DEBUG }, 17:55:08 DEBUG "value": [ 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG [...] 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:55:08 DEBUG }, 17:55:08 DEBUG "value": [ 17:55:08 DEBUG 1713545708, 17:55:08 DEBUG "0.026" 17:55:08 DEBUG ] 17:55:08 DEBUG } 17:55:08 DEBUG ] 17:55:08 DEBUG } 17:55:08 DEBUG } 17:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545708" 17:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545708" 17:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545708" 17:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545708" 17:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545708" 17:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545708" 17:55:08 INFO [http_cmd]: http status code OK 17:55:08 INFO [http_cmd]: http status code OK 17:55:08 DEBUG --- status code --- 17:55:08 INFO [http_cmd]: http status code OK 17:55:08 DEBUG --- status code --- 17:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:08 DEBUG --- status code --- 17:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:08 INFO [http_cmd]: http status code OK 17:55:08 DEBUG --- http response --- 17:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:08 DEBUG --- http response --- 17:55:08 DEBUG --- status code --- 17:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:55:08 DEBUG --- http response --- 17:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:55:08 INFO [http_cmd]: http status code OK 17:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:08 DEBUG ----- output ----- 17:55:08 DEBUG ----- output ----- 17:55:08 DEBUG --- status code --- 17:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:55:08 DEBUG --- http response --- 17:55:08 DEBUG { 17:55:08 DEBUG { 17:55:08 INFO [http_cmd]: http status code OK 17:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:08 DEBUG ----- output ----- 17:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:55:08 DEBUG "status": "success", 17:55:08 DEBUG "status": "success", 17:55:08 DEBUG --- status code --- 17:55:08 DEBUG --- http response --- 17:55:08 DEBUG { 17:55:08 DEBUG ----- output ----- 17:55:08 DEBUG "data": { 17:55:08 DEBUG "data": { 17:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:55:08 DEBUG "status": "success", 17:55:08 DEBUG { 17:55:08 DEBUG "resultType": "vector", 17:55:08 DEBUG "resultType": "vector", 17:55:08 DEBUG --- http response --- 17:55:08 DEBUG ----- output ----- 17:55:08 DEBUG "data": { 17:55:08 DEBUG "status": "success", 17:55:08 DEBUG "result": [ 17:55:08 DEBUG "result": [ 17:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:55:08 DEBUG { 17:55:08 DEBUG "resultType": "vector", 17:55:08 DEBUG "data": { 17:55:08 DEBUG { 17:55:08 DEBUG { 17:55:08 DEBUG ----- output ----- 17:55:08 DEBUG "status": "success", 17:55:08 DEBUG "result": [ 17:55:08 DEBUG "resultType": "vector", 17:55:08 DEBUG "metric": { 17:55:08 DEBUG "metric": { 17:55:08 DEBUG { 17:55:08 DEBUG "data": { 17:55:08 DEBUG { 17:55:08 DEBUG "result": [ 17:55:08 DEBUG "instance": "10.142.0.107:9100" 17:55:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:55:08 DEBUG "status": "success", 17:55:08 DEBUG "resultType": "vector", 17:55:08 DEBUG "metric": { 17:55:08 DEBUG { 17:55:08 DEBUG }, 17:55:08 DEBUG }, 17:55:08 DEBUG "data": { 17:55:08 DEBUG "result": [ 17:55:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:55:08 DEBUG "metric": { 17:55:08 DEBUG "value": [ 17:55:08 DEBUG "value": [ 17:55:08 DEBUG "resultType": "vector", 17:55:08 DEBUG { 17:55:08 DEBUG }, 17:55:08 DEBUG "pod": "ds-idrepo-0" 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG "result": [ 17:55:08 DEBUG "metric": { 17:55:08 DEBUG "value": [ 17:55:08 DEBUG }, 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG { 17:55:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:55:08 DEBUG 17:55:08 DEBUG "value": [ 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG "metric": { 17:55:08 DEBUG }, 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG [...] 17:55:08 DEBUG [...] 17:55:08 DEBUG "pod": "ds-cts-0" 17:55:08 DEBUG "value": [ 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG }, 17:55:08 DEBUG 17:55:08 DEBUG [...] 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG "value": [ 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG [...] 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG "instance": "10.142.0.43:9100" 17:55:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:55:08 DEBUG 17:55:08 DEBUG [...] 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG }, 17:55:08 DEBUG }, 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:55:08 DEBUG 17:55:08 DEBUG "value": [ 17:55:08 DEBUG "value": [ 17:55:08 DEBUG [...] 17:55:08 DEBUG 17:55:08 DEBUG }, 17:55:08 DEBUG "pod": "ds-idrepo-2" 17:55:08 DEBUG 1713545708, 17:55:08 DEBUG 1713545708, 17:55:08 DEBUG 17:55:08 DEBUG 17:55:08 DEBUG "value": [ 17:55:08 DEBUG }, 17:55:08 DEBUG "0.36003333333338217" 17:55:08 DEBUG "0" 17:55:08 DEBUG 17:55:08 DEBUG "pod": "ds-idrepo-2" 17:55:08 DEBUG 1713545708, 17:55:08 DEBUG "value": [ 17:55:08 DEBUG ] 17:55:08 DEBUG ] 17:55:08 DEBUG 17:55:08 DEBUG }, 17:55:08 DEBUG "1.0247457560380897" 17:55:08 DEBUG 1713545708, 17:55:08 DEBUG } 17:55:08 DEBUG } 17:55:08 DEBUG "pod": "ds-cts-2" 17:55:08 DEBUG "value": [ 17:55:08 DEBUG ] 17:55:08 DEBUG "0" 17:55:08 DEBUG ] 17:55:08 DEBUG ] 17:55:08 DEBUG }, 17:55:08 DEBUG 1713545708, 17:55:08 DEBUG } 17:55:08 DEBUG ] 17:55:08 DEBUG } 17:55:08 DEBUG } 17:55:08 DEBUG "value": [ 17:55:08 DEBUG "0" 17:55:08 DEBUG ] 17:55:08 DEBUG } 17:55:08 DEBUG } 17:55:08 DEBUG } 17:55:08 DEBUG 1713545708, 17:55:08 DEBUG ] 17:55:08 DEBUG } 17:55:08 DEBUG ] 17:55:08 DEBUG "608.7333333333333" 17:55:08 DEBUG } 17:55:08 DEBUG } 17:55:08 DEBUG } 17:55:08 DEBUG ] 17:55:08 DEBUG ] 17:55:08 DEBUG } 17:55:08 DEBUG } 17:55:08 DEBUG } 17:55:08 DEBUG ] 17:55:08 DEBUG } 17:55:08 DEBUG } 17:55:08 DEBUG } 17:55:09 INFO 17:55:09 INFO [loop_until]: kubectl --namespace=xlou top pods 17:55:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:55:09 INFO [loop_until]: OK (rc = 0) 17:55:09 DEBUG --- stdout --- 17:55:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 2186m 8904Mi am-6fd554f8f7-btd4s 2103m 8910Mi am-6fd554f8f7-wxcsj 2169m 8922Mi ds-cts-0 9233m 13230Mi ds-cts-1 8836m 13531Mi ds-cts-2 9851m 13209Mi ds-idrepo-0 647m 3452Mi ds-idrepo-1 638m 3411Mi ds-idrepo-2 654m 3468Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1028m 879Mi 17:55:09 DEBUG --- stderr --- 17:55:09 DEBUG 17:55:09 INFO 17:55:09 INFO [loop_until]: kubectl --namespace=xlou top node 17:55:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:55:10 INFO [loop_until]: OK (rc = 0) 17:55:10 DEBUG --- stdout --- 17:55:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6185Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 750m 2% 7019Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 50m 0% 2681Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 2206m 7% 12211Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9232m 30% 16596Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 136m 0% 5221Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2195m 7% 11836Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 81m 0% 3243Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2315m 7% 11183Mi 10% gke-xlou-cdm-ds-0f601751-j488 733m 2% 6687Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 9544m 31% 17895Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 708m 2% 7256Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9556m 32% 15779Mi 14% gke-xlou-cdm-frontend-a8771548-i82n 1075m 6% 4697Mi 8% 17:55:10 DEBUG --- stderr --- 17:55:10 DEBUG 17:56:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545765" 17:56:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545765" 17:56:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545765" 17:56:05 INFO [http_cmd]: http status code OK 17:56:05 DEBUG --- status code --- 17:56:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:05 DEBUG --- http response --- 17:56:05 INFO [http_cmd]: http status code OK 17:56:05 ERROR [http_cmd]: ERROR 17:56:05 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:56:05 DEBUG --- status code --- 17:56:05 ERROR --- status code --- 17:56:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:05 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:56:05 DEBUG --- http response --- 17:56:05 ERROR --- http response --- 17:56:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:56:05 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:56:05 DEBUG ----- output ----- 17:56:05 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:56:05 DEBUG { 17:56:05 DEBUG "status": "success", 17:56:05 DEBUG "data": { 17:56:05 DEBUG "resultType": "vector", 17:56:05 DEBUG "result": [ 17:56:05 DEBUG { 17:56:05 DEBUG "metric": { 17:56:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:56:05 DEBUG }, 17:56:05 DEBUG "value": [ 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG [...] 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:56:05 DEBUG }, 17:56:05 DEBUG "value": [ 17:56:05 DEBUG 1713545765, 17:56:05 DEBUG "2340.7" 17:56:05 DEBUG ] 17:56:05 DEBUG } 17:56:05 DEBUG ] 17:56:05 DEBUG } 17:56:05 DEBUG } 17:56:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545765" 17:56:05 INFO [http_cmd]: http status code OK 17:56:05 DEBUG --- status code --- 17:56:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:05 DEBUG --- http response --- 17:56:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:56:05 DEBUG ----- output ----- 17:56:05 DEBUG { 17:56:05 DEBUG "status": "success", 17:56:05 DEBUG "data": { 17:56:05 DEBUG "resultType": "vector", 17:56:05 DEBUG "result": [ 17:56:05 DEBUG { 17:56:05 DEBUG "metric": { 17:56:05 DEBUG "pod": "ds-idrepo-0" 17:56:05 DEBUG }, 17:56:05 DEBUG "value": [ 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG [...] 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG "pod": "ds-cts-2" 17:56:05 DEBUG }, 17:56:05 DEBUG "value": [ 17:56:05 DEBUG 1713545765, 17:56:05 DEBUG "0" 17:56:05 DEBUG ] 17:56:05 DEBUG } 17:56:05 DEBUG ] 17:56:05 DEBUG } 17:56:05 DEBUG } 17:56:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545765" 17:56:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545765" 17:56:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545765" 17:56:05 INFO [http_cmd]: http status code OK 17:56:05 DEBUG --- status code --- 17:56:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:05 DEBUG --- http response --- 17:56:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:56:05 DEBUG ----- output ----- 17:56:05 DEBUG { 17:56:05 DEBUG "status": "success", 17:56:05 DEBUG "data": { 17:56:05 DEBUG "resultType": "vector", 17:56:05 DEBUG "result": [ 17:56:05 DEBUG { 17:56:05 DEBUG "metric": { 17:56:05 INFO [http_cmd]: http status code OK 17:56:05 INFO [http_cmd]: http status code OK 17:56:05 DEBUG "pod": "ds-idrepo-1" 17:56:05 DEBUG --- status code --- 17:56:05 DEBUG --- status code --- 17:56:05 DEBUG }, 17:56:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:05 DEBUG "value": [ 17:56:05 DEBUG --- http response --- 17:56:05 DEBUG --- http response --- 17:56:05 DEBUG 17:56:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:56:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:56:05 DEBUG 17:56:05 DEBUG ----- output ----- 17:56:05 DEBUG ----- output ----- 17:56:05 DEBUG 17:56:05 DEBUG { 17:56:05 DEBUG { 17:56:05 DEBUG [...] 17:56:05 DEBUG "status": "success", 17:56:05 DEBUG "status": "success", 17:56:05 DEBUG 17:56:05 DEBUG "data": { 17:56:05 DEBUG "data": { 17:56:05 DEBUG 17:56:05 DEBUG "resultType": "vector", 17:56:05 DEBUG "resultType": "vector", 17:56:05 DEBUG 17:56:05 DEBUG "result": [ 17:56:05 DEBUG "result": [ 17:56:05 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:56:05 DEBUG { 17:56:05 DEBUG { 17:56:05 DEBUG }, 17:56:05 DEBUG "metric": { 17:56:05 DEBUG "metric": { 17:56:05 DEBUG "value": [ 17:56:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:56:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:56:05 DEBUG 1713545765, 17:56:05 DEBUG }, 17:56:05 DEBUG }, 17:56:05 DEBUG "0" 17:56:05 DEBUG "value": [ 17:56:05 DEBUG "value": [ 17:56:05 DEBUG ] 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG } 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG ] 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG } 17:56:05 DEBUG [...] 17:56:05 DEBUG [...] 17:56:05 DEBUG } 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:56:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:56:05 DEBUG }, 17:56:05 DEBUG }, 17:56:05 DEBUG "value": [ 17:56:05 DEBUG "value": [ 17:56:05 DEBUG 1713545765, 17:56:05 DEBUG 1713545765, 17:56:05 DEBUG "0" 17:56:05 DEBUG "0" 17:56:05 DEBUG ] 17:56:05 DEBUG ] 17:56:05 DEBUG } 17:56:05 DEBUG } 17:56:05 DEBUG ] 17:56:05 DEBUG ] 17:56:05 DEBUG } 17:56:05 DEBUG } 17:56:05 DEBUG } 17:56:05 DEBUG } 17:56:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545765" 17:56:05 INFO [http_cmd]: http status code OK 17:56:05 DEBUG --- status code --- 17:56:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:05 DEBUG --- http response --- 17:56:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:56:05 DEBUG ----- output ----- 17:56:05 DEBUG { 17:56:05 DEBUG "status": "success", 17:56:05 DEBUG "data": { 17:56:05 DEBUG "resultType": "vector", 17:56:05 DEBUG "result": [ 17:56:05 DEBUG { 17:56:05 DEBUG "metric": { 17:56:05 DEBUG "instance": "10.142.0.107:9100" 17:56:05 DEBUG }, 17:56:05 DEBUG "value": [ 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG [...] 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG 17:56:05 DEBUG "instance": "10.142.0.43:9100" 17:56:05 DEBUG }, 17:56:05 DEBUG "value": [ 17:56:05 DEBUG 1713545765, 17:56:05 DEBUG "0.6429999999999895" 17:56:05 DEBUG ] 17:56:05 DEBUG } 17:56:05 DEBUG ] 17:56:05 DEBUG } 17:56:05 DEBUG } 17:56:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545766" 17:56:06 INFO [http_cmd]: http status code OK 17:56:06 DEBUG --- status code --- 17:56:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:06 DEBUG --- http response --- 17:56:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:56:06 DEBUG ----- output ----- 17:56:06 DEBUG { 17:56:06 DEBUG "status": "success", 17:56:06 DEBUG "data": { 17:56:06 DEBUG "resultType": "vector", 17:56:06 DEBUG "result": [ 17:56:06 DEBUG { 17:56:06 DEBUG "metric": { 17:56:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:56:06 DEBUG }, 17:56:06 DEBUG "value": [ 17:56:06 DEBUG 17:56:06 DEBUG 17:56:06 DEBUG 17:56:06 DEBUG [...] 17:56:06 DEBUG 17:56:06 DEBUG 17:56:06 DEBUG 17:56:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:56:06 DEBUG }, 17:56:06 DEBUG "value": [ 17:56:06 DEBUG 1713545766, 17:56:06 DEBUG "3562692" 17:56:06 DEBUG ] 17:56:06 DEBUG } 17:56:06 DEBUG ] 17:56:06 DEBUG } 17:56:06 DEBUG } 17:56:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545766" 17:56:06 INFO [http_cmd]: http status code OK 17:56:06 DEBUG --- status code --- 17:56:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:06 DEBUG --- http response --- 17:56:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:56:06 DEBUG ----- output ----- 17:56:06 DEBUG { 17:56:06 DEBUG "status": "success", 17:56:06 DEBUG "data": { 17:56:06 DEBUG "resultType": "vector", 17:56:06 DEBUG "result": [ 17:56:06 DEBUG { 17:56:06 DEBUG "metric": { 17:56:06 DEBUG "instance": "10.142.0.107:9100" 17:56:06 DEBUG }, 17:56:06 DEBUG "value": [ 17:56:06 DEBUG 17:56:06 DEBUG 17:56:06 DEBUG 17:56:06 DEBUG [...] 17:56:06 DEBUG 17:56:06 DEBUG 17:56:06 DEBUG 17:56:06 DEBUG "instance": "10.142.0.43:9100" 17:56:06 DEBUG }, 17:56:06 DEBUG "value": [ 17:56:06 DEBUG 1713545766, 17:56:06 DEBUG "1802147951616" 17:56:06 DEBUG ] 17:56:06 DEBUG } 17:56:06 DEBUG ] 17:56:06 DEBUG } 17:56:06 DEBUG } 17:56:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545767" 17:56:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545767" 17:56:07 INFO [http_cmd]: http status code OK 17:56:07 DEBUG --- status code --- 17:56:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:07 DEBUG --- http response --- 17:56:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:56:07 DEBUG ----- output ----- 17:56:07 DEBUG { 17:56:07 DEBUG "status": "success", 17:56:07 DEBUG "data": { 17:56:07 DEBUG "resultType": "vector", 17:56:07 DEBUG "result": [ 17:56:07 DEBUG { 17:56:07 DEBUG "metric": { 17:56:07 DEBUG "pod": "ds-idrepo-0" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG [...] 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG "pod": "ds-idrepo-2" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 1713545767, 17:56:07 DEBUG "0" 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG } 17:56:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545767" 17:56:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545767" 17:56:07 INFO [http_cmd]: http status code OK 17:56:07 DEBUG --- status code --- 17:56:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:07 DEBUG --- http response --- 17:56:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:56:07 DEBUG ----- output ----- 17:56:07 DEBUG { 17:56:07 DEBUG "status": "success", 17:56:07 DEBUG "data": { 17:56:07 DEBUG "resultType": "vector", 17:56:07 DEBUG "result": [ 17:56:07 DEBUG { 17:56:07 DEBUG "metric": { 17:56:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG [...] 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 1713545767, 17:56:07 DEBUG "1274580992" 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG } 17:56:07 INFO [http_cmd]: http status code OK 17:56:07 INFO [http_cmd]: http status code OK 17:56:07 DEBUG --- status code --- 17:56:07 DEBUG --- status code --- 17:56:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:07 DEBUG --- http response --- 17:56:07 DEBUG --- http response --- 17:56:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:56:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:56:07 DEBUG ----- output ----- 17:56:07 DEBUG ----- output ----- 17:56:07 DEBUG { 17:56:07 DEBUG { 17:56:07 DEBUG "status": "success", 17:56:07 DEBUG "status": "success", 17:56:07 DEBUG "data": { 17:56:07 DEBUG "data": { 17:56:07 DEBUG "resultType": "vector", 17:56:07 DEBUG "resultType": "vector", 17:56:07 DEBUG "result": [ 17:56:07 DEBUG "result": [ 17:56:07 DEBUG { 17:56:07 DEBUG { 17:56:07 DEBUG "metric": { 17:56:07 DEBUG "metric": { 17:56:07 DEBUG "pod": "ds-cts-0" 17:56:07 DEBUG "pod": "ds-cts-0" 17:56:07 DEBUG }, 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG [...] 17:56:07 DEBUG [...] 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG "pod": "ds-cts-2" 17:56:07 DEBUG "pod": "ds-cts-2" 17:56:07 DEBUG }, 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 1713545767, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG "13.033333333333333" 17:56:07 DEBUG 1713545767, 17:56:07 DEBUG ] 17:56:07 DEBUG "0.0008009266666666666" 17:56:07 DEBUG } 17:56:07 DEBUG ] 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG } 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG } 17:56:07 DEBUG } 17:56:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545767" 17:56:07 INFO [http_cmd]: http status code OK 17:56:07 DEBUG --- status code --- 17:56:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:07 DEBUG --- http response --- 17:56:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:56:07 DEBUG ----- output ----- 17:56:07 DEBUG { 17:56:07 DEBUG "status": "success", 17:56:07 DEBUG "data": { 17:56:07 DEBUG "resultType": "vector", 17:56:07 DEBUG "result": [ 17:56:07 DEBUG { 17:56:07 DEBUG "metric": { 17:56:07 DEBUG "pod": "ds-idrepo-0" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG [...] 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG "pod": "ds-idrepo-2" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 1713545767, 17:56:07 DEBUG "0" 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG } 17:56:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545767" 17:56:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545767" 17:56:07 INFO [http_cmd]: http status code OK 17:56:07 DEBUG --- status code --- 17:56:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:07 DEBUG --- http response --- 17:56:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:56:07 DEBUG ----- output ----- 17:56:07 DEBUG { 17:56:07 DEBUG "status": "success", 17:56:07 DEBUG "data": { 17:56:07 DEBUG "resultType": "vector", 17:56:07 DEBUG "result": [ 17:56:07 DEBUG { 17:56:07 DEBUG "metric": { 17:56:07 DEBUG "instance": "10.142.0.107:9100" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG [...] 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG "instance": "10.142.0.43:9100" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 1713545767, 17:56:07 DEBUG "16231108.066666666" 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG } 17:56:07 INFO [http_cmd]: http status code OK 17:56:07 DEBUG --- status code --- 17:56:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:07 DEBUG --- http response --- 17:56:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:56:07 DEBUG ----- output ----- 17:56:07 DEBUG { 17:56:07 DEBUG "status": "success", 17:56:07 DEBUG "data": { 17:56:07 DEBUG "resultType": "vector", 17:56:07 DEBUG "result": [ 17:56:07 DEBUG { 17:56:07 DEBUG "metric": { 17:56:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG [...] 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 1713545767, 17:56:07 DEBUG "1232.8272356364123" 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG } 17:56:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545767" 17:56:07 INFO [http_cmd]: http status code OK 17:56:07 DEBUG --- status code --- 17:56:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:07 DEBUG --- http response --- 17:56:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:56:07 DEBUG ----- output ----- 17:56:07 DEBUG { 17:56:07 DEBUG "status": "success", 17:56:07 DEBUG "data": { 17:56:07 DEBUG "resultType": "vector", 17:56:07 DEBUG "result": [ 17:56:07 DEBUG { 17:56:07 DEBUG "metric": { 17:56:07 DEBUG "instance": "10.142.0.107:9100" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG [...] 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG "instance": "10.142.0.43:9100" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 1713545767, 17:56:07 DEBUG "16221673.666666666" 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG } 17:56:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545767" 17:56:07 INFO [http_cmd]: http status code OK 17:56:07 DEBUG --- status code --- 17:56:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:07 DEBUG --- http response --- 17:56:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:56:07 DEBUG ----- output ----- 17:56:07 DEBUG { 17:56:07 DEBUG "status": "success", 17:56:07 DEBUG "data": { 17:56:07 DEBUG "resultType": "vector", 17:56:07 DEBUG "result": [ 17:56:07 DEBUG { 17:56:07 DEBUG "metric": { 17:56:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG [...] 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG 17:56:07 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:56:07 DEBUG }, 17:56:07 DEBUG "value": [ 17:56:07 DEBUG 1713545767, 17:56:07 DEBUG "0" 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG ] 17:56:07 DEBUG } 17:56:07 DEBUG } 17:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545768" 17:56:08 INFO [http_cmd]: http status code OK 17:56:08 DEBUG --- status code --- 17:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:08 DEBUG --- http response --- 17:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:56:08 DEBUG ----- output ----- 17:56:08 DEBUG { 17:56:08 DEBUG "status": "success", 17:56:08 DEBUG "data": { 17:56:08 DEBUG "resultType": "vector", 17:56:08 DEBUG "result": [ 17:56:08 DEBUG { 17:56:08 DEBUG "metric": { 17:56:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:56:08 DEBUG }, 17:56:08 DEBUG "value": [ 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG [...] 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:56:08 DEBUG }, 17:56:08 DEBUG "value": [ 17:56:08 DEBUG 1713545768, 17:56:08 DEBUG "0" 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG } 17:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545768" 17:56:08 INFO [http_cmd]: http status code OK 17:56:08 DEBUG --- status code --- 17:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:08 DEBUG --- http response --- 17:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:56:08 DEBUG ----- output ----- 17:56:08 DEBUG { 17:56:08 DEBUG "status": "success", 17:56:08 DEBUG "data": { 17:56:08 DEBUG "resultType": "vector", 17:56:08 DEBUG "result": [ 17:56:08 DEBUG { 17:56:08 DEBUG "metric": { 17:56:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:56:08 DEBUG }, 17:56:08 DEBUG "value": [ 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG [...] 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:56:08 DEBUG }, 17:56:08 DEBUG "value": [ 17:56:08 DEBUG 1713545768, 17:56:08 DEBUG "0.026" 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG } 17:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545768" 17:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545768" 17:56:08 INFO [http_cmd]: http status code OK 17:56:08 DEBUG --- status code --- 17:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:08 DEBUG --- http response --- 17:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:56:08 DEBUG ----- output ----- 17:56:08 DEBUG { 17:56:08 DEBUG "status": "success", 17:56:08 DEBUG "data": { 17:56:08 DEBUG "resultType": "vector", 17:56:08 DEBUG "result": [ 17:56:08 DEBUG { 17:56:08 DEBUG "metric": { 17:56:08 DEBUG "instance": "10.142.0.107:9100" 17:56:08 DEBUG }, 17:56:08 DEBUG "value": [ 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG [...] 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG "instance": "10.142.0.43:9100" 17:56:08 DEBUG }, 17:56:08 DEBUG "value": [ 17:56:08 DEBUG 1713545768, 17:56:08 DEBUG "0.6901666666666841" 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG } 17:56:08 INFO [http_cmd]: http status code OK 17:56:08 DEBUG --- status code --- 17:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:08 DEBUG --- http response --- 17:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:56:08 DEBUG ----- output ----- 17:56:08 DEBUG { 17:56:08 DEBUG "status": "success", 17:56:08 DEBUG "data": { 17:56:08 DEBUG "resultType": "vector", 17:56:08 DEBUG "result": [ 17:56:08 DEBUG { 17:56:08 DEBUG "metric": { 17:56:08 DEBUG "pod": "ds-cts-0" 17:56:08 DEBUG }, 17:56:08 DEBUG "value": [ 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG [...] 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG "pod": "ds-cts-2" 17:56:08 DEBUG }, 17:56:08 DEBUG "value": [ 17:56:08 DEBUG 1713545768, 17:56:08 DEBUG "279.56666666666666" 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG } 17:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545768" 17:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545768" 17:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545768" 17:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545768" 17:56:08 INFO [http_cmd]: http status code OK 17:56:08 DEBUG --- status code --- 17:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:08 DEBUG --- http response --- 17:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:56:08 DEBUG ----- output ----- 17:56:08 DEBUG { 17:56:08 DEBUG "status": "success", 17:56:08 DEBUG "data": { 17:56:08 DEBUG "resultType": "vector", 17:56:08 DEBUG "result": [ 17:56:08 DEBUG { 17:56:08 DEBUG "metric": { 17:56:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:56:08 DEBUG }, 17:56:08 DEBUG "value": [ 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG [...] 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:56:08 DEBUG }, 17:56:08 DEBUG "value": [ 17:56:08 DEBUG 1713545768, 17:56:08 DEBUG "0" 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG } 17:56:08 INFO [http_cmd]: http status code OK 17:56:08 DEBUG --- status code --- 17:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:08 DEBUG --- http response --- 17:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:56:08 DEBUG ----- output ----- 17:56:08 DEBUG { 17:56:08 INFO [http_cmd]: http status code OK 17:56:08 DEBUG "status": "success", 17:56:08 DEBUG --- status code --- 17:56:08 DEBUG "data": { 17:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:08 DEBUG "resultType": "vector", 17:56:08 INFO [http_cmd]: http status code OK 17:56:08 DEBUG --- http response --- 17:56:08 DEBUG "result": [ 17:56:08 DEBUG --- status code --- 17:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:56:08 DEBUG { 17:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:56:08 DEBUG ----- output ----- 17:56:08 DEBUG { 17:56:08 DEBUG --- http response --- 17:56:08 DEBUG "status": "success", 17:56:08 DEBUG "metric": { 17:56:08 DEBUG "data": { 17:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:56:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:56:08 DEBUG "resultType": "vector", 17:56:08 DEBUG ----- output ----- 17:56:08 DEBUG { 17:56:08 DEBUG "result": [ 17:56:08 DEBUG "status": "success", 17:56:08 DEBUG }, 17:56:08 DEBUG { 17:56:08 DEBUG "data": { 17:56:08 DEBUG "value": [ 17:56:08 DEBUG "metric": { 17:56:08 DEBUG "resultType": "vector", 17:56:08 DEBUG 17:56:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:56:08 DEBUG "result": [ 17:56:08 DEBUG 17:56:08 DEBUG }, 17:56:08 DEBUG { 17:56:08 DEBUG 17:56:08 DEBUG "value": [ 17:56:08 DEBUG "metric": { 17:56:08 DEBUG [...] 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG "pod": "ds-idrepo-0" 17:56:08 DEBUG [...] 17:56:08 DEBUG 17:56:08 DEBUG }, 17:56:08 DEBUG 17:56:08 DEBUG "pod": "ds-cts-1" 17:56:08 DEBUG "value": [ 17:56:08 DEBUG 17:56:08 DEBUG }, 17:56:08 DEBUG 17:56:08 DEBUG 17:56:08 DEBUG "value": [ 17:56:08 DEBUG 17:56:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:56:08 DEBUG 1713545768, 17:56:08 DEBUG 17:56:08 DEBUG }, 17:56:08 DEBUG "9.998721164087861" 17:56:08 DEBUG [...] 17:56:08 DEBUG "value": [ 17:56:08 DEBUG ] 17:56:08 DEBUG 17:56:08 DEBUG 1713545768, 17:56:08 DEBUG } 17:56:08 DEBUG 17:56:08 DEBUG "1.0314398688816915" 17:56:08 DEBUG ] 17:56:08 DEBUG 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG "pod": "ds-idrepo-2" 17:56:08 DEBUG } 17:56:08 DEBUG } 17:56:08 DEBUG }, 17:56:08 DEBUG ] 17:56:08 DEBUG "value": [ 17:56:08 DEBUG } 17:56:08 DEBUG 1713545768, 17:56:08 DEBUG } 17:56:08 DEBUG "0" 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG ] 17:56:08 DEBUG } 17:56:08 DEBUG } 17:56:09 INFO 17:56:09 INFO [loop_until]: kubectl --namespace=xlou top pods 17:56:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:56:09 INFO [loop_until]: OK (rc = 0) 17:56:09 DEBUG --- stdout --- 17:56:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 2174m 8906Mi am-6fd554f8f7-btd4s 2157m 8908Mi am-6fd554f8f7-wxcsj 2084m 8923Mi ds-cts-0 9307m 13379Mi ds-cts-1 10011m 12884Mi ds-cts-2 9523m 13161Mi ds-idrepo-0 674m 3451Mi ds-idrepo-1 643m 3410Mi ds-idrepo-2 646m 3468Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1011m 1208Mi 17:56:09 DEBUG --- stderr --- 17:56:09 DEBUG 17:56:10 INFO 17:56:10 INFO [loop_until]: kubectl --namespace=xlou top node 17:56:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:56:10 INFO [loop_until]: OK (rc = 0) 17:56:10 DEBUG --- stdout --- 17:56:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6183Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 743m 2% 7030Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 47m 0% 2683Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 2241m 7% 12213Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9146m 30% 16277Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5233Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2226m 7% 11836Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 77m 0% 3231Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2334m 7% 11169Mi 10% gke-xlou-cdm-ds-0f601751-j488 717m 2% 6693Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 10158m 34% 17716Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 766m 2% 7251Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9713m 32% 15628Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 1075m 6% 5065Mi 8% 17:56:10 DEBUG --- stderr --- 17:56:10 DEBUG 17:57:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545825" 17:57:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545825" 17:57:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545825" 17:57:05 ERROR [http_cmd]: ERROR 17:57:05 INFO [http_cmd]: http status code OK 17:57:05 ERROR --- status code --- 17:57:05 INFO [http_cmd]: http status code OK 17:57:05 DEBUG --- status code --- 17:57:05 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:57:05 DEBUG --- status code --- 17:57:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:05 ERROR --- http response --- 17:57:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:05 DEBUG --- http response --- 17:57:05 DEBUG --- http response --- 17:57:05 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:57:05 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:57:05 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:57:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:57:05 DEBUG ----- output ----- 17:57:05 DEBUG { 17:57:05 DEBUG "status": "success", 17:57:05 DEBUG "data": { 17:57:05 DEBUG "resultType": "vector", 17:57:05 DEBUG "result": [ 17:57:05 DEBUG { 17:57:05 DEBUG "metric": { 17:57:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:57:05 DEBUG }, 17:57:05 DEBUG "value": [ 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG [...] 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:57:05 DEBUG }, 17:57:05 DEBUG "value": [ 17:57:05 DEBUG 1713545825, 17:57:05 DEBUG "2338.5" 17:57:05 DEBUG ] 17:57:05 DEBUG } 17:57:05 DEBUG ] 17:57:05 DEBUG } 17:57:05 DEBUG } 17:57:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545825" 17:57:05 INFO [http_cmd]: http status code OK 17:57:05 DEBUG --- status code --- 17:57:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:05 DEBUG --- http response --- 17:57:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:57:05 DEBUG ----- output ----- 17:57:05 DEBUG { 17:57:05 DEBUG "status": "success", 17:57:05 DEBUG "data": { 17:57:05 DEBUG "resultType": "vector", 17:57:05 DEBUG "result": [ 17:57:05 DEBUG { 17:57:05 DEBUG "metric": { 17:57:05 DEBUG "pod": "ds-idrepo-0" 17:57:05 DEBUG }, 17:57:05 DEBUG "value": [ 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG [...] 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG "pod": "ds-cts-2" 17:57:05 DEBUG }, 17:57:05 DEBUG "value": [ 17:57:05 DEBUG 1713545825, 17:57:05 DEBUG "0" 17:57:05 DEBUG ] 17:57:05 DEBUG } 17:57:05 DEBUG ] 17:57:05 DEBUG } 17:57:05 DEBUG } 17:57:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545825" 17:57:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545825" 17:57:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545825" 17:57:05 INFO [http_cmd]: http status code OK 17:57:05 DEBUG --- status code --- 17:57:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:05 DEBUG --- http response --- 17:57:05 INFO [http_cmd]: http status code OK 17:57:05 DEBUG --- status code --- 17:57:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:05 DEBUG --- http response --- 17:57:05 INFO [http_cmd]: http status code OK 17:57:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:57:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:57:05 DEBUG --- status code --- 17:57:05 DEBUG ----- output ----- 17:57:05 DEBUG ----- output ----- 17:57:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:05 DEBUG { 17:57:05 DEBUG { 17:57:05 DEBUG --- http response --- 17:57:05 DEBUG "status": "success", 17:57:05 DEBUG "status": "success", 17:57:05 DEBUG "data": { 17:57:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:57:05 DEBUG "data": { 17:57:05 DEBUG "resultType": "vector", 17:57:05 DEBUG ----- output ----- 17:57:05 DEBUG "resultType": "vector", 17:57:05 DEBUG "result": [ 17:57:05 DEBUG { 17:57:05 DEBUG "result": [ 17:57:05 DEBUG { 17:57:05 DEBUG "status": "success", 17:57:05 DEBUG { 17:57:05 DEBUG "metric": { 17:57:05 DEBUG "data": { 17:57:05 DEBUG "metric": { 17:57:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:57:05 DEBUG "resultType": "vector", 17:57:05 DEBUG "pod": "ds-idrepo-0" 17:57:05 DEBUG }, 17:57:05 DEBUG "result": [ 17:57:05 DEBUG }, 17:57:05 DEBUG "value": [ 17:57:05 DEBUG { 17:57:05 DEBUG "value": [ 17:57:05 DEBUG 17:57:05 DEBUG "metric": { 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG }, 17:57:05 DEBUG 17:57:05 DEBUG [...] 17:57:05 DEBUG "value": [ 17:57:05 DEBUG [...] 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG 17:57:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:57:05 DEBUG [...] 17:57:05 DEBUG "pod": "idm-b85f9d978-txpwv" 17:57:05 DEBUG }, 17:57:05 DEBUG 17:57:05 DEBUG }, 17:57:05 DEBUG "value": [ 17:57:05 DEBUG 17:57:05 DEBUG "value": [ 17:57:05 DEBUG 1713545825, 17:57:05 DEBUG 17:57:05 DEBUG 1713545825, 17:57:05 DEBUG "0" 17:57:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:57:05 DEBUG "0" 17:57:05 DEBUG ] 17:57:05 DEBUG }, 17:57:05 DEBUG ] 17:57:05 DEBUG } 17:57:05 DEBUG "value": [ 17:57:05 DEBUG } 17:57:05 DEBUG ] 17:57:05 DEBUG 1713545825, 17:57:05 DEBUG ] 17:57:05 DEBUG } 17:57:05 DEBUG "0" 17:57:05 DEBUG } 17:57:05 DEBUG } 17:57:05 DEBUG ] 17:57:05 DEBUG } 17:57:05 DEBUG } 17:57:05 DEBUG ] 17:57:05 DEBUG } 17:57:05 DEBUG } 17:57:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545826" 17:57:06 INFO [http_cmd]: http status code OK 17:57:06 DEBUG --- status code --- 17:57:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:06 DEBUG --- http response --- 17:57:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:57:06 DEBUG ----- output ----- 17:57:06 DEBUG { 17:57:06 DEBUG "status": "success", 17:57:06 DEBUG "data": { 17:57:06 DEBUG "resultType": "vector", 17:57:06 DEBUG "result": [ 17:57:06 DEBUG { 17:57:06 DEBUG "metric": { 17:57:06 DEBUG "instance": "10.142.0.107:9100" 17:57:06 DEBUG }, 17:57:06 DEBUG "value": [ 17:57:06 DEBUG 17:57:06 DEBUG 17:57:06 DEBUG 17:57:06 DEBUG [...] 17:57:06 DEBUG 17:57:06 DEBUG 17:57:06 DEBUG 17:57:06 DEBUG "instance": "10.142.0.43:9100" 17:57:06 DEBUG }, 17:57:06 DEBUG "value": [ 17:57:06 DEBUG 1713545826, 17:57:06 DEBUG "0.45366666666667094" 17:57:06 DEBUG ] 17:57:06 DEBUG } 17:57:06 DEBUG ] 17:57:06 DEBUG } 17:57:06 DEBUG } 17:57:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545826" 17:57:06 INFO [http_cmd]: http status code OK 17:57:06 DEBUG --- status code --- 17:57:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:06 DEBUG --- http response --- 17:57:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:57:06 DEBUG ----- output ----- 17:57:06 DEBUG { 17:57:06 DEBUG "status": "success", 17:57:06 DEBUG "data": { 17:57:06 DEBUG "resultType": "vector", 17:57:06 DEBUG "result": [ 17:57:06 DEBUG { 17:57:06 DEBUG "metric": { 17:57:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:57:06 DEBUG }, 17:57:06 DEBUG "value": [ 17:57:06 DEBUG 17:57:06 DEBUG 17:57:06 DEBUG 17:57:06 DEBUG [...] 17:57:06 DEBUG 17:57:06 DEBUG 17:57:06 DEBUG 17:57:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:57:06 DEBUG }, 17:57:06 DEBUG "value": [ 17:57:06 DEBUG 1713545826, 17:57:06 DEBUG "3422518.5" 17:57:06 DEBUG ] 17:57:06 DEBUG } 17:57:06 DEBUG ] 17:57:06 DEBUG } 17:57:06 DEBUG } 17:57:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545826" 17:57:06 INFO [http_cmd]: http status code OK 17:57:06 DEBUG --- status code --- 17:57:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:06 DEBUG --- http response --- 17:57:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:57:06 DEBUG ----- output ----- 17:57:06 DEBUG { 17:57:06 DEBUG "status": "success", 17:57:06 DEBUG "data": { 17:57:06 DEBUG "resultType": "vector", 17:57:06 DEBUG "result": [ 17:57:06 DEBUG { 17:57:06 DEBUG "metric": { 17:57:06 DEBUG "instance": "10.142.0.107:9100" 17:57:06 DEBUG }, 17:57:06 DEBUG "value": [ 17:57:06 DEBUG 17:57:06 DEBUG 17:57:06 DEBUG 17:57:06 DEBUG [...] 17:57:06 DEBUG 17:57:06 DEBUG 17:57:06 DEBUG 17:57:06 DEBUG "instance": "10.142.0.43:9100" 17:57:06 DEBUG }, 17:57:06 DEBUG "value": [ 17:57:06 DEBUG 1713545826, 17:57:06 DEBUG "1803776121856" 17:57:06 DEBUG ] 17:57:06 DEBUG } 17:57:06 DEBUG ] 17:57:06 DEBUG } 17:57:06 DEBUG } 17:57:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545827" 17:57:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545827" 17:57:07 INFO [http_cmd]: http status code OK 17:57:07 INFO [http_cmd]: http status code OK 17:57:07 DEBUG --- status code --- 17:57:07 DEBUG --- status code --- 17:57:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:07 DEBUG --- http response --- 17:57:07 DEBUG --- http response --- 17:57:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:57:07 DEBUG ----- output ----- 17:57:07 DEBUG { 17:57:07 DEBUG "status": "success", 17:57:07 DEBUG "data": { 17:57:07 DEBUG "resultType": "vector", 17:57:07 DEBUG "result": [ 17:57:07 DEBUG { 17:57:07 DEBUG "metric": { 17:57:07 DEBUG "pod": "ds-idrepo-0" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG [...] 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG "pod": "ds-idrepo-2" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 1713545827, 17:57:07 DEBUG "0" 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG } 17:57:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545827" 17:57:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:57:07 DEBUG ----- output ----- 17:57:07 DEBUG { 17:57:07 DEBUG "status": "success", 17:57:07 DEBUG "data": { 17:57:07 DEBUG "resultType": "vector", 17:57:07 DEBUG "result": [ 17:57:07 DEBUG { 17:57:07 DEBUG "metric": { 17:57:07 DEBUG "pod": "ds-cts-0" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG [...] 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG "pod": "ds-cts-2" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 1713545827, 17:57:07 DEBUG "0.0005815666666666667" 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG } 17:57:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545827" 17:57:07 INFO [http_cmd]: http status code OK 17:57:07 DEBUG --- status code --- 17:57:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:07 DEBUG --- http response --- 17:57:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:57:07 DEBUG ----- output ----- 17:57:07 DEBUG { 17:57:07 DEBUG "status": "success", 17:57:07 DEBUG "data": { 17:57:07 DEBUG "resultType": "vector", 17:57:07 DEBUG "result": [ 17:57:07 DEBUG { 17:57:07 DEBUG "metric": { 17:57:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG [...] 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 1713545827, 17:57:07 DEBUG "1652592640" 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG } 17:57:07 INFO [http_cmd]: http status code OK 17:57:07 DEBUG --- status code --- 17:57:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:07 DEBUG --- http response --- 17:57:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:57:07 DEBUG ----- output ----- 17:57:07 DEBUG { 17:57:07 DEBUG "status": "success", 17:57:07 DEBUG "data": { 17:57:07 DEBUG "resultType": "vector", 17:57:07 DEBUG "result": [ 17:57:07 DEBUG { 17:57:07 DEBUG "metric": { 17:57:07 DEBUG "pod": "ds-cts-0" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG [...] 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG "pod": "ds-cts-2" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 1713545827, 17:57:07 DEBUG "10.533333333333333" 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG } 17:57:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545827" 17:57:07 INFO [http_cmd]: http status code OK 17:57:07 DEBUG --- status code --- 17:57:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:07 DEBUG --- http response --- 17:57:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:57:07 DEBUG ----- output ----- 17:57:07 DEBUG { 17:57:07 DEBUG "status": "success", 17:57:07 DEBUG "data": { 17:57:07 DEBUG "resultType": "vector", 17:57:07 DEBUG "result": [ 17:57:07 DEBUG { 17:57:07 DEBUG "metric": { 17:57:07 DEBUG "pod": "ds-idrepo-0" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG [...] 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG "pod": "ds-idrepo-2" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 1713545827, 17:57:07 DEBUG "0" 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG } 17:57:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545827" 17:57:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545827" 17:57:07 INFO [http_cmd]: http status code OK 17:57:07 DEBUG --- status code --- 17:57:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:07 DEBUG --- http response --- 17:57:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:57:07 DEBUG ----- output ----- 17:57:07 DEBUG { 17:57:07 INFO [http_cmd]: http status code OK 17:57:07 DEBUG "status": "success", 17:57:07 DEBUG --- status code --- 17:57:07 DEBUG "data": { 17:57:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:07 DEBUG --- http response --- 17:57:07 DEBUG "resultType": "vector", 17:57:07 DEBUG "result": [ 17:57:07 DEBUG { 17:57:07 DEBUG "metric": { 17:57:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG ----- output ----- 17:57:07 DEBUG 17:57:07 DEBUG { 17:57:07 DEBUG [...] 17:57:07 DEBUG "status": "success", 17:57:07 DEBUG 17:57:07 DEBUG "data": { 17:57:07 DEBUG 17:57:07 DEBUG "resultType": "vector", 17:57:07 DEBUG 17:57:07 DEBUG "result": [ 17:57:07 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:57:07 DEBUG { 17:57:07 DEBUG }, 17:57:07 DEBUG "metric": { 17:57:07 DEBUG "value": [ 17:57:07 DEBUG "instance": "10.142.0.107:9100" 17:57:07 DEBUG 1713545827, 17:57:07 DEBUG }, 17:57:07 DEBUG "1198.2767786416312" 17:57:07 DEBUG "value": [ 17:57:07 DEBUG ] 17:57:07 DEBUG 17:57:07 DEBUG } 17:57:07 DEBUG 17:57:07 DEBUG ] 17:57:07 DEBUG 17:57:07 DEBUG } 17:57:07 DEBUG [...] 17:57:07 DEBUG } 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG "instance": "10.142.0.43:9100" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 1713545827, 17:57:07 DEBUG "16247694.433333332" 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG } 17:57:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545827" 17:57:07 INFO [http_cmd]: http status code OK 17:57:07 DEBUG --- status code --- 17:57:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:07 DEBUG --- http response --- 17:57:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:57:07 DEBUG ----- output ----- 17:57:07 DEBUG { 17:57:07 DEBUG "status": "success", 17:57:07 DEBUG "data": { 17:57:07 DEBUG "resultType": "vector", 17:57:07 DEBUG "result": [ 17:57:07 DEBUG { 17:57:07 DEBUG "metric": { 17:57:07 DEBUG "instance": "10.142.0.107:9100" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG [...] 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG 17:57:07 DEBUG "instance": "10.142.0.43:9100" 17:57:07 DEBUG }, 17:57:07 DEBUG "value": [ 17:57:07 DEBUG 1713545827, 17:57:07 DEBUG "16237819.5" 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG ] 17:57:07 DEBUG } 17:57:07 DEBUG } 17:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545828" 17:57:08 INFO [http_cmd]: http status code OK 17:57:08 DEBUG --- status code --- 17:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:08 DEBUG --- http response --- 17:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:57:08 DEBUG ----- output ----- 17:57:08 DEBUG { 17:57:08 DEBUG "status": "success", 17:57:08 DEBUG "data": { 17:57:08 DEBUG "resultType": "vector", 17:57:08 DEBUG "result": [ 17:57:08 DEBUG { 17:57:08 DEBUG "metric": { 17:57:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG [...] 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 1713545828, 17:57:08 DEBUG "0" 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG } 17:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545828" 17:57:08 INFO [http_cmd]: http status code OK 17:57:08 DEBUG --- status code --- 17:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:08 DEBUG --- http response --- 17:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:57:08 DEBUG ----- output ----- 17:57:08 DEBUG { 17:57:08 DEBUG "status": "success", 17:57:08 DEBUG "data": { 17:57:08 DEBUG "resultType": "vector", 17:57:08 DEBUG "result": [ 17:57:08 DEBUG { 17:57:08 DEBUG "metric": { 17:57:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG [...] 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 1713545828, 17:57:08 DEBUG "0" 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG } 17:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545828" 17:57:08 INFO [http_cmd]: http status code OK 17:57:08 DEBUG --- status code --- 17:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:08 DEBUG --- http response --- 17:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:57:08 DEBUG ----- output ----- 17:57:08 DEBUG { 17:57:08 DEBUG "status": "success", 17:57:08 DEBUG "data": { 17:57:08 DEBUG "resultType": "vector", 17:57:08 DEBUG "result": [ 17:57:08 DEBUG { 17:57:08 DEBUG "metric": { 17:57:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG [...] 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 1713545828, 17:57:08 DEBUG "0.026" 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG } 17:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545828" 17:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545828" 17:57:08 INFO [http_cmd]: http status code OK 17:57:08 DEBUG --- status code --- 17:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:08 DEBUG --- http response --- 17:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:57:08 DEBUG ----- output ----- 17:57:08 DEBUG { 17:57:08 DEBUG "status": "success", 17:57:08 DEBUG "data": { 17:57:08 DEBUG "resultType": "vector", 17:57:08 DEBUG "result": [ 17:57:08 DEBUG { 17:57:08 DEBUG "metric": { 17:57:08 DEBUG "instance": "10.142.0.107:9100" 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG [...] 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG "instance": "10.142.0.43:9100" 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 1713545828, 17:57:08 DEBUG "0.5638666666667357" 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG } 17:57:08 INFO [http_cmd]: http status code OK 17:57:08 DEBUG --- status code --- 17:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:08 DEBUG --- http response --- 17:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:57:08 DEBUG ----- output ----- 17:57:08 DEBUG { 17:57:08 DEBUG "status": "success", 17:57:08 DEBUG "data": { 17:57:08 DEBUG "resultType": "vector", 17:57:08 DEBUG "result": [ 17:57:08 DEBUG { 17:57:08 DEBUG "metric": { 17:57:08 DEBUG "pod": "ds-cts-0" 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG [...] 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG "pod": "ds-cts-2" 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 1713545828, 17:57:08 DEBUG "492.7" 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG } 17:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545828" 17:57:08 INFO [http_cmd]: http status code OK 17:57:08 DEBUG --- status code --- 17:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:08 DEBUG --- http response --- 17:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:57:08 DEBUG ----- output ----- 17:57:08 DEBUG { 17:57:08 DEBUG "status": "success", 17:57:08 DEBUG "data": { 17:57:08 DEBUG "resultType": "vector", 17:57:08 DEBUG "result": [ 17:57:08 DEBUG { 17:57:08 DEBUG "metric": { 17:57:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG [...] 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG 17:57:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 1713545828, 17:57:08 DEBUG "0" 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG } 17:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545828" 17:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545828" 17:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545828" 17:57:08 INFO [http_cmd]: http status code OK 17:57:08 DEBUG --- status code --- 17:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:08 DEBUG --- http response --- 17:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:57:08 DEBUG ----- output ----- 17:57:08 DEBUG { 17:57:08 DEBUG "status": "success", 17:57:08 DEBUG "data": { 17:57:08 DEBUG "resultType": "vector", 17:57:08 DEBUG "result": [ 17:57:08 DEBUG { 17:57:08 INFO [http_cmd]: http status code OK 17:57:08 DEBUG "metric": { 17:57:08 DEBUG --- status code --- 17:57:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:57:08 DEBUG }, 17:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:08 DEBUG "value": [ 17:57:08 DEBUG --- http response --- 17:57:08 DEBUG 17:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:57:08 DEBUG 17:57:08 DEBUG ----- output ----- 17:57:08 DEBUG 17:57:08 DEBUG { 17:57:08 DEBUG [...] 17:57:08 DEBUG "status": "success", 17:57:08 INFO [http_cmd]: http status code OK 17:57:08 DEBUG 17:57:08 DEBUG "data": { 17:57:08 DEBUG --- status code --- 17:57:08 DEBUG 17:57:08 DEBUG "resultType": "vector", 17:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:57:08 DEBUG 17:57:08 DEBUG "result": [ 17:57:08 DEBUG --- http response --- 17:57:08 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:57:08 DEBUG { 17:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:57:08 DEBUG }, 17:57:08 DEBUG "metric": { 17:57:08 DEBUG ----- output ----- 17:57:08 DEBUG "value": [ 17:57:08 DEBUG "pod": "ds-idrepo-0" 17:57:08 DEBUG { 17:57:08 DEBUG 1713545828, 17:57:08 DEBUG }, 17:57:08 DEBUG "status": "success", 17:57:08 DEBUG "0.003199729049924246" 17:57:08 DEBUG "value": [ 17:57:08 DEBUG "data": { 17:57:08 DEBUG ] 17:57:08 DEBUG 17:57:08 DEBUG "resultType": "vector", 17:57:08 DEBUG } 17:57:08 DEBUG 17:57:08 DEBUG "result": [ 17:57:08 DEBUG ] 17:57:08 DEBUG 17:57:08 DEBUG { 17:57:08 DEBUG } 17:57:08 DEBUG [...] 17:57:08 DEBUG "metric": { 17:57:08 DEBUG } 17:57:08 DEBUG 17:57:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:57:08 DEBUG 17:57:08 DEBUG }, 17:57:08 DEBUG 17:57:08 DEBUG "value": [ 17:57:08 DEBUG "pod": "ds-idrepo-2" 17:57:08 DEBUG 17:57:08 DEBUG }, 17:57:08 DEBUG 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 17:57:08 DEBUG 1713545828, 17:57:08 DEBUG [...] 17:57:08 DEBUG "0" 17:57:08 DEBUG ] 17:57:08 DEBUG 17:57:08 DEBUG } 17:57:08 DEBUG 17:57:08 DEBUG ] 17:57:08 DEBUG 17:57:08 DEBUG } 17:57:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:57:08 DEBUG } 17:57:08 DEBUG }, 17:57:08 DEBUG "value": [ 17:57:08 DEBUG 1713545828, 17:57:08 DEBUG "1.0323733526820316" 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG ] 17:57:08 DEBUG } 17:57:08 DEBUG } 17:57:09 INFO 17:57:09 INFO [loop_until]: kubectl --namespace=xlou top pods 17:57:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:57:09 INFO [loop_until]: OK (rc = 0) 17:57:09 DEBUG --- stdout --- 17:57:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 2227m 8904Mi am-6fd554f8f7-btd4s 2067m 8903Mi am-6fd554f8f7-wxcsj 2114m 8923Mi ds-cts-0 9409m 13738Mi ds-cts-1 9210m 12872Mi ds-cts-2 10129m 12755Mi ds-idrepo-0 650m 3451Mi ds-idrepo-1 650m 3411Mi ds-idrepo-2 637m 3472Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1174Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 5Mi overseer-0-5c67bf94db-77mh9 1011m 1663Mi 17:57:09 DEBUG --- stderr --- 17:57:09 DEBUG 17:57:10 INFO 17:57:10 INFO [loop_until]: kubectl --namespace=xlou top node 17:57:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:57:10 INFO [loop_until]: OK (rc = 0) 17:57:10 DEBUG --- stdout --- 17:57:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6168Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 732m 2% 7013Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 49m 0% 2683Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 2229m 7% 12213Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9573m 32% 16704Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 148m 0% 5239Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2156m 7% 11819Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 76m 0% 3199Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2309m 7% 11182Mi 10% gke-xlou-cdm-ds-0f601751-j488 732m 2% 6710Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 9448m 31% 17737Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 762m 2% 7234Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 10080m 33% 15318Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 1076m 6% 5454Mi 9% 17:57:10 DEBUG --- stderr --- 17:57:10 DEBUG 17:58:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545885" 17:58:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545885" 17:58:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545885" 17:58:05 INFO [http_cmd]: http status code OK 17:58:05 DEBUG --- status code --- 17:58:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:05 DEBUG --- http response --- 17:58:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:58:05 DEBUG ----- output ----- 17:58:05 DEBUG { 17:58:05 DEBUG "status": "success", 17:58:05 DEBUG "data": { 17:58:05 DEBUG "resultType": "vector", 17:58:05 DEBUG "result": [ 17:58:05 DEBUG { 17:58:05 DEBUG "metric": { 17:58:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:58:05 DEBUG }, 17:58:05 DEBUG "value": [ 17:58:05 DEBUG 17:58:05 DEBUG 17:58:05 DEBUG 17:58:05 DEBUG [...] 17:58:05 DEBUG 17:58:05 DEBUG 17:58:05 DEBUG 17:58:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:58:05 DEBUG }, 17:58:05 DEBUG "value": [ 17:58:05 DEBUG 1713545885, 17:58:05 DEBUG "2337.6666666666665" 17:58:05 DEBUG ] 17:58:05 DEBUG } 17:58:05 DEBUG ] 17:58:05 DEBUG } 17:58:05 ERROR [http_cmd]: ERROR 17:58:05 DEBUG } 17:58:05 INFO [http_cmd]: http status code OK 17:58:05 ERROR --- status code --- 17:58:05 DEBUG --- status code --- 17:58:05 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:58:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:05 ERROR --- http response --- 17:58:05 DEBUG --- http response --- 17:58:05 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:58:05 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:58:05 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:58:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545885" 17:58:06 INFO [http_cmd]: http status code OK 17:58:06 DEBUG --- status code --- 17:58:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:06 DEBUG --- http response --- 17:58:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:58:06 DEBUG ----- output ----- 17:58:06 DEBUG { 17:58:06 DEBUG "status": "success", 17:58:06 DEBUG "data": { 17:58:06 DEBUG "resultType": "vector", 17:58:06 DEBUG "result": [ 17:58:06 DEBUG { 17:58:06 DEBUG "metric": { 17:58:06 DEBUG "pod": "ds-idrepo-0" 17:58:06 DEBUG }, 17:58:06 DEBUG "value": [ 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG [...] 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG "pod": "ds-cts-2" 17:58:06 DEBUG }, 17:58:06 DEBUG "value": [ 17:58:06 DEBUG 1713545885, 17:58:06 DEBUG "0" 17:58:06 DEBUG ] 17:58:06 DEBUG } 17:58:06 DEBUG ] 17:58:06 DEBUG } 17:58:06 DEBUG } 17:58:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545886" 17:58:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545886" 17:58:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545886" 17:58:06 INFO [http_cmd]: http status code OK 17:58:06 DEBUG --- status code --- 17:58:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:06 DEBUG --- http response --- 17:58:06 INFO [http_cmd]: http status code OK 17:58:06 INFO [http_cmd]: http status code OK 17:58:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:58:06 DEBUG --- status code --- 17:58:06 DEBUG --- status code --- 17:58:06 DEBUG ----- output ----- 17:58:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:06 DEBUG { 17:58:06 DEBUG --- http response --- 17:58:06 DEBUG --- http response --- 17:58:06 DEBUG "status": "success", 17:58:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:58:06 DEBUG "data": { 17:58:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:58:06 DEBUG ----- output ----- 17:58:06 DEBUG "resultType": "vector", 17:58:06 DEBUG ----- output ----- 17:58:06 DEBUG { 17:58:06 DEBUG "result": [ 17:58:06 DEBUG { 17:58:06 DEBUG "status": "success", 17:58:06 DEBUG { 17:58:06 DEBUG "status": "success", 17:58:06 DEBUG "data": { 17:58:06 DEBUG "metric": { 17:58:06 DEBUG "data": { 17:58:06 DEBUG "resultType": "vector", 17:58:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:58:06 DEBUG "resultType": "vector", 17:58:06 DEBUG "result": [ 17:58:06 DEBUG }, 17:58:06 DEBUG "result": [ 17:58:06 DEBUG { 17:58:06 DEBUG "value": [ 17:58:06 DEBUG { 17:58:06 DEBUG "metric": { 17:58:06 DEBUG 17:58:06 DEBUG "metric": { 17:58:06 DEBUG "pod": "ds-idrepo-0" 17:58:06 DEBUG 17:58:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:58:06 DEBUG }, 17:58:06 DEBUG 17:58:06 DEBUG }, 17:58:06 DEBUG "value": [ 17:58:06 DEBUG [...] 17:58:06 DEBUG "value": [ 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG [...] 17:58:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:58:06 DEBUG [...] 17:58:06 DEBUG 17:58:06 DEBUG }, 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG "value": [ 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 1713545886, 17:58:06 DEBUG 17:58:06 DEBUG "pod": "ds-idrepo-2" 17:58:06 DEBUG "0" 17:58:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:58:06 DEBUG }, 17:58:06 DEBUG ] 17:58:06 DEBUG }, 17:58:06 DEBUG "value": [ 17:58:06 DEBUG } 17:58:06 DEBUG "value": [ 17:58:06 DEBUG 1713545886, 17:58:06 DEBUG ] 17:58:06 DEBUG 1713545886, 17:58:06 DEBUG "0" 17:58:06 DEBUG } 17:58:06 DEBUG "0" 17:58:06 DEBUG ] 17:58:06 DEBUG } 17:58:06 DEBUG ] 17:58:06 DEBUG } 17:58:06 DEBUG } 17:58:06 DEBUG ] 17:58:06 DEBUG ] 17:58:06 DEBUG } 17:58:06 DEBUG } 17:58:06 DEBUG } 17:58:06 DEBUG } 17:58:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545886" 17:58:06 INFO [http_cmd]: http status code OK 17:58:06 DEBUG --- status code --- 17:58:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:06 DEBUG --- http response --- 17:58:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:58:06 DEBUG ----- output ----- 17:58:06 DEBUG { 17:58:06 DEBUG "status": "success", 17:58:06 DEBUG "data": { 17:58:06 DEBUG "resultType": "vector", 17:58:06 DEBUG "result": [ 17:58:06 DEBUG { 17:58:06 DEBUG "metric": { 17:58:06 DEBUG "instance": "10.142.0.107:9100" 17:58:06 DEBUG }, 17:58:06 DEBUG "value": [ 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG [...] 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG "instance": "10.142.0.43:9100" 17:58:06 DEBUG }, 17:58:06 DEBUG "value": [ 17:58:06 DEBUG 1713545886, 17:58:06 DEBUG "0.43533333333333907" 17:58:06 DEBUG ] 17:58:06 DEBUG } 17:58:06 DEBUG ] 17:58:06 DEBUG } 17:58:06 DEBUG } 17:58:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545886" 17:58:06 INFO [http_cmd]: http status code OK 17:58:06 DEBUG --- status code --- 17:58:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:06 DEBUG --- http response --- 17:58:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:58:06 DEBUG ----- output ----- 17:58:06 DEBUG { 17:58:06 DEBUG "status": "success", 17:58:06 DEBUG "data": { 17:58:06 DEBUG "resultType": "vector", 17:58:06 DEBUG "result": [ 17:58:06 DEBUG { 17:58:06 DEBUG "metric": { 17:58:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:58:06 DEBUG }, 17:58:06 DEBUG "value": [ 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG [...] 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:58:06 DEBUG }, 17:58:06 DEBUG "value": [ 17:58:06 DEBUG 1713545886, 17:58:06 DEBUG "3282336" 17:58:06 DEBUG ] 17:58:06 DEBUG } 17:58:06 DEBUG ] 17:58:06 DEBUG } 17:58:06 DEBUG } 17:58:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545886" 17:58:06 INFO [http_cmd]: http status code OK 17:58:06 DEBUG --- status code --- 17:58:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:06 DEBUG --- http response --- 17:58:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:58:06 DEBUG ----- output ----- 17:58:06 DEBUG { 17:58:06 DEBUG "status": "success", 17:58:06 DEBUG "data": { 17:58:06 DEBUG "resultType": "vector", 17:58:06 DEBUG "result": [ 17:58:06 DEBUG { 17:58:06 DEBUG "metric": { 17:58:06 DEBUG "instance": "10.142.0.107:9100" 17:58:06 DEBUG }, 17:58:06 DEBUG "value": [ 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG [...] 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG 17:58:06 DEBUG "instance": "10.142.0.43:9100" 17:58:06 DEBUG }, 17:58:06 DEBUG "value": [ 17:58:06 DEBUG 1713545886, 17:58:06 DEBUG "1806331313152" 17:58:06 DEBUG ] 17:58:06 DEBUG } 17:58:06 DEBUG ] 17:58:06 DEBUG } 17:58:06 DEBUG } 17:58:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545887" 17:58:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545887" 17:58:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545887" 17:58:07 INFO [http_cmd]: http status code OK 17:58:07 INFO [http_cmd]: http status code OK 17:58:07 DEBUG --- status code --- 17:58:07 DEBUG --- status code --- 17:58:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:07 DEBUG --- http response --- 17:58:07 DEBUG --- http response --- 17:58:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:58:07 DEBUG ----- output ----- 17:58:07 DEBUG { 17:58:07 DEBUG "status": "success", 17:58:07 DEBUG "data": { 17:58:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:58:07 DEBUG "resultType": "vector", 17:58:07 DEBUG ----- output ----- 17:58:07 DEBUG "result": [ 17:58:07 DEBUG { 17:58:07 DEBUG { 17:58:07 DEBUG "status": "success", 17:58:07 DEBUG "metric": { 17:58:07 DEBUG "data": { 17:58:07 DEBUG "pod": "ds-idrepo-0" 17:58:07 DEBUG "resultType": "vector", 17:58:07 DEBUG }, 17:58:07 DEBUG "result": [ 17:58:07 DEBUG "value": [ 17:58:07 DEBUG { 17:58:07 DEBUG 17:58:07 DEBUG "metric": { 17:58:07 DEBUG 17:58:07 DEBUG "pod": "ds-cts-0" 17:58:07 DEBUG 17:58:07 DEBUG }, 17:58:07 DEBUG [...] 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG "pod": "ds-idrepo-2" 17:58:07 DEBUG [...] 17:58:07 DEBUG }, 17:58:07 DEBUG 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 17:58:07 DEBUG 1713545887, 17:58:07 DEBUG 17:58:07 DEBUG "0" 17:58:07 DEBUG "pod": "ds-cts-2" 17:58:07 DEBUG ] 17:58:07 DEBUG }, 17:58:07 DEBUG } 17:58:07 DEBUG "value": [ 17:58:07 DEBUG ] 17:58:07 DEBUG 1713545887, 17:58:07 DEBUG } 17:58:07 DEBUG "0.0004752533333333333" 17:58:07 DEBUG } 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG } 17:58:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545887" 17:58:07 INFO [http_cmd]: http status code OK 17:58:07 DEBUG --- status code --- 17:58:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:07 DEBUG --- http response --- 17:58:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:58:07 DEBUG ----- output ----- 17:58:07 DEBUG { 17:58:07 DEBUG "status": "success", 17:58:07 DEBUG "data": { 17:58:07 DEBUG "resultType": "vector", 17:58:07 DEBUG "result": [ 17:58:07 DEBUG { 17:58:07 DEBUG "metric": { 17:58:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:58:07 DEBUG }, 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG [...] 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:58:07 DEBUG }, 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 1713545887, 17:58:07 DEBUG "1721487360" 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG } 17:58:07 INFO [http_cmd]: http status code OK 17:58:07 DEBUG --- status code --- 17:58:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:07 DEBUG --- http response --- 17:58:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:58:07 DEBUG ----- output ----- 17:58:07 DEBUG { 17:58:07 DEBUG "status": "success", 17:58:07 DEBUG "data": { 17:58:07 DEBUG "resultType": "vector", 17:58:07 DEBUG "result": [ 17:58:07 DEBUG { 17:58:07 DEBUG "metric": { 17:58:07 DEBUG "pod": "ds-cts-0" 17:58:07 DEBUG }, 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG [...] 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG "pod": "ds-cts-2" 17:58:07 DEBUG }, 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 1713545887, 17:58:07 DEBUG "12.2" 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG } 17:58:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545887" 17:58:07 INFO [http_cmd]: http status code OK 17:58:07 DEBUG --- status code --- 17:58:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:07 DEBUG --- http response --- 17:58:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:58:07 DEBUG ----- output ----- 17:58:07 DEBUG { 17:58:07 DEBUG "status": "success", 17:58:07 DEBUG "data": { 17:58:07 DEBUG "resultType": "vector", 17:58:07 DEBUG "result": [ 17:58:07 DEBUG { 17:58:07 DEBUG "metric": { 17:58:07 DEBUG "pod": "ds-idrepo-0" 17:58:07 DEBUG }, 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG [...] 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG "pod": "ds-idrepo-2" 17:58:07 DEBUG }, 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 1713545887, 17:58:07 DEBUG "0" 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG } 17:58:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545887" 17:58:07 INFO [http_cmd]: http status code OK 17:58:07 DEBUG --- status code --- 17:58:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:07 DEBUG --- http response --- 17:58:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:58:07 DEBUG ----- output ----- 17:58:07 DEBUG { 17:58:07 DEBUG "status": "success", 17:58:07 DEBUG "data": { 17:58:07 DEBUG "resultType": "vector", 17:58:07 DEBUG "result": [ 17:58:07 DEBUG { 17:58:07 DEBUG "metric": { 17:58:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:58:07 DEBUG }, 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG [...] 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG "pod": "ds-cts-1" 17:58:07 DEBUG }, 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 1713545887, 17:58:07 DEBUG "33147935.460104614" 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG } 17:58:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545887" 17:58:07 INFO [http_cmd]: http status code OK 17:58:07 DEBUG --- status code --- 17:58:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:07 DEBUG --- http response --- 17:58:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:58:07 DEBUG ----- output ----- 17:58:07 DEBUG { 17:58:07 DEBUG "status": "success", 17:58:07 DEBUG "data": { 17:58:07 DEBUG "resultType": "vector", 17:58:07 DEBUG "result": [ 17:58:07 DEBUG { 17:58:07 DEBUG "metric": { 17:58:07 DEBUG "instance": "10.142.0.107:9100" 17:58:07 DEBUG }, 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG [...] 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG "instance": "10.142.0.43:9100" 17:58:07 DEBUG }, 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 1713545887, 17:58:07 DEBUG "16185422.766666666" 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG } 17:58:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545887" 17:58:07 INFO [http_cmd]: http status code OK 17:58:07 DEBUG --- status code --- 17:58:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:07 DEBUG --- http response --- 17:58:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:58:07 DEBUG ----- output ----- 17:58:07 DEBUG { 17:58:07 DEBUG "status": "success", 17:58:07 DEBUG "data": { 17:58:07 DEBUG "resultType": "vector", 17:58:07 DEBUG "result": [ 17:58:07 DEBUG { 17:58:07 DEBUG "metric": { 17:58:07 DEBUG "instance": "10.142.0.107:9100" 17:58:07 DEBUG }, 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG [...] 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG 17:58:07 DEBUG "instance": "10.142.0.43:9100" 17:58:07 DEBUG }, 17:58:07 DEBUG "value": [ 17:58:07 DEBUG 1713545887, 17:58:07 DEBUG "16176307.133333333" 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG ] 17:58:07 DEBUG } 17:58:07 DEBUG } 17:58:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545888" 17:58:08 INFO [http_cmd]: http status code OK 17:58:08 DEBUG --- status code --- 17:58:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:08 DEBUG --- http response --- 17:58:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:58:08 DEBUG ----- output ----- 17:58:08 DEBUG { 17:58:08 DEBUG "status": "success", 17:58:08 DEBUG "data": { 17:58:08 DEBUG "resultType": "vector", 17:58:08 DEBUG "result": [ 17:58:08 DEBUG { 17:58:08 DEBUG "metric": { 17:58:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG [...] 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG "pod": "ds-cts-1" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 1713545888, 17:58:08 DEBUG "65287940.47005083" 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG } 17:58:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545888" 17:58:08 INFO [http_cmd]: http status code OK 17:58:08 DEBUG --- status code --- 17:58:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:08 DEBUG --- http response --- 17:58:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:58:08 DEBUG ----- output ----- 17:58:08 DEBUG { 17:58:08 DEBUG "status": "success", 17:58:08 DEBUG "data": { 17:58:08 DEBUG "resultType": "vector", 17:58:08 DEBUG "result": [ 17:58:08 DEBUG { 17:58:08 DEBUG "metric": { 17:58:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG [...] 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 1713545888, 17:58:08 DEBUG "0" 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG } 17:58:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545888" 17:58:08 INFO [http_cmd]: http status code OK 17:58:08 DEBUG --- status code --- 17:58:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:08 DEBUG --- http response --- 17:58:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:58:08 DEBUG ----- output ----- 17:58:08 DEBUG { 17:58:08 DEBUG "status": "success", 17:58:08 DEBUG "data": { 17:58:08 DEBUG "resultType": "vector", 17:58:08 DEBUG "result": [ 17:58:08 DEBUG { 17:58:08 DEBUG "metric": { 17:58:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG [...] 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 1713545888, 17:58:08 DEBUG "0.0265" 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG } 17:58:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545888" 17:58:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545888" 17:58:08 INFO [http_cmd]: http status code OK 17:58:08 INFO [http_cmd]: http status code OK 17:58:08 DEBUG --- status code --- 17:58:08 DEBUG --- status code --- 17:58:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:08 DEBUG --- http response --- 17:58:08 DEBUG --- http response --- 17:58:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:58:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:58:08 DEBUG ----- output ----- 17:58:08 DEBUG ----- output ----- 17:58:08 DEBUG { 17:58:08 DEBUG { 17:58:08 DEBUG "status": "success", 17:58:08 DEBUG "status": "success", 17:58:08 DEBUG "data": { 17:58:08 DEBUG "data": { 17:58:08 DEBUG "resultType": "vector", 17:58:08 DEBUG "resultType": "vector", 17:58:08 DEBUG "result": [ 17:58:08 DEBUG "result": [ 17:58:08 DEBUG { 17:58:08 DEBUG { 17:58:08 DEBUG "metric": { 17:58:08 DEBUG "metric": { 17:58:08 DEBUG "pod": "ds-cts-0" 17:58:08 DEBUG "instance": "10.142.0.107:9100" 17:58:08 DEBUG }, 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG [...] 17:58:08 DEBUG [...] 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG "pod": "ds-cts-2" 17:58:08 DEBUG "instance": "10.142.0.43:9100" 17:58:08 DEBUG }, 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 1713545888, 17:58:08 DEBUG 1713545888, 17:58:08 DEBUG "272.3666666666667" 17:58:08 DEBUG "0.5575666666665569" 17:58:08 DEBUG ] 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG } 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG } 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG } 17:58:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545888" 17:58:08 INFO [http_cmd]: http status code OK 17:58:08 DEBUG --- status code --- 17:58:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:08 DEBUG --- http response --- 17:58:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:58:08 DEBUG ----- output ----- 17:58:08 DEBUG { 17:58:08 DEBUG "status": "success", 17:58:08 DEBUG "data": { 17:58:08 DEBUG "resultType": "vector", 17:58:08 DEBUG "result": [ 17:58:08 DEBUG { 17:58:08 DEBUG "metric": { 17:58:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG [...] 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 1713545888, 17:58:08 DEBUG "0" 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG } 17:58:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545888" 17:58:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545888" 17:58:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545888" 17:58:08 INFO [http_cmd]: http status code OK 17:58:08 DEBUG --- status code --- 17:58:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:08 DEBUG --- http response --- 17:58:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:58:08 DEBUG ----- output ----- 17:58:08 DEBUG { 17:58:08 DEBUG "status": "success", 17:58:08 DEBUG "data": { 17:58:08 DEBUG "resultType": "vector", 17:58:08 DEBUG "result": [ 17:58:08 DEBUG { 17:58:08 DEBUG "metric": { 17:58:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG [...] 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG "pod": "ds-cts-1" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 1713545888, 17:58:08 DEBUG "9.446800928313374" 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG } 17:58:08 INFO [http_cmd]: http status code OK 17:58:08 DEBUG --- status code --- 17:58:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:08 DEBUG --- http response --- 17:58:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:58:08 DEBUG ----- output ----- 17:58:08 DEBUG { 17:58:08 DEBUG "status": "success", 17:58:08 DEBUG "data": { 17:58:08 DEBUG "resultType": "vector", 17:58:08 DEBUG "result": [ 17:58:08 DEBUG { 17:58:08 DEBUG "metric": { 17:58:08 DEBUG "pod": "ds-idrepo-0" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG [...] 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG "pod": "ds-idrepo-2" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 1713545888, 17:58:08 DEBUG "0" 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG } 17:58:08 INFO [http_cmd]: http status code OK 17:58:08 DEBUG --- status code --- 17:58:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:58:08 DEBUG --- http response --- 17:58:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:58:08 DEBUG ----- output ----- 17:58:08 DEBUG { 17:58:08 DEBUG "status": "success", 17:58:08 DEBUG "data": { 17:58:08 DEBUG "resultType": "vector", 17:58:08 DEBUG "result": [ 17:58:08 DEBUG { 17:58:08 DEBUG "metric": { 17:58:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG [...] 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG 17:58:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:58:08 DEBUG }, 17:58:08 DEBUG "value": [ 17:58:08 DEBUG 1713545888, 17:58:08 DEBUG "0.8707980120158945" 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG ] 17:58:08 DEBUG } 17:58:08 DEBUG } 17:58:09 INFO 17:58:09 INFO [loop_until]: kubectl --namespace=xlou top pods 17:58:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:58:09 INFO [loop_until]: OK (rc = 0) 17:58:09 DEBUG --- stdout --- 17:58:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 2221m 8904Mi am-6fd554f8f7-btd4s 2071m 8904Mi am-6fd554f8f7-wxcsj 2071m 8929Mi ds-cts-0 9617m 13307Mi ds-cts-1 9737m 12963Mi ds-cts-2 10095m 12946Mi ds-idrepo-0 703m 3452Mi ds-idrepo-1 642m 3410Mi ds-idrepo-2 665m 3472Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 3m 1217Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 200m 1549Mi 17:58:09 DEBUG --- stderr --- 17:58:09 DEBUG 17:58:10 INFO 17:58:10 INFO [loop_until]: kubectl --namespace=xlou top node 17:58:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:58:10 INFO [loop_until]: OK (rc = 0) 17:58:10 DEBUG --- stdout --- 17:58:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 60m 0% 6179Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 732m 2% 7013Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 49m 0% 2678Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 2212m 7% 12214Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 9615m 32% 16259Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5233Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2210m 7% 11829Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 75m 0% 3201Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2358m 7% 11184Mi 10% gke-xlou-cdm-ds-0f601751-j488 711m 2% 6700Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 9577m 32% 17812Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 759m 2% 7237Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 9944m 33% 15487Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 284m 1% 5378Mi 9% 17:58:10 DEBUG --- stderr --- 17:58:10 DEBUG 17:59:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713545945" 17:59:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545945" 17:59:05 ERROR [http_cmd]: ERROR 17:59:05 INFO [http_cmd]: http status code OK 17:59:05 ERROR --- status code --- 17:59:05 DEBUG --- status code --- 17:59:05 ERROR http status code is 400 (expected [200, 401, 403, 404]) 17:59:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:05 ERROR --- http response --- 17:59:05 DEBUG --- http response --- 17:59:05 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 17:59:05 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 17:59:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:59:05 DEBUG ----- output ----- 17:59:05 DEBUG { 17:59:05 DEBUG "status": "success", 17:59:05 DEBUG "data": { 17:59:05 DEBUG "resultType": "vector", 17:59:05 DEBUG "result": [ 17:59:05 DEBUG { 17:59:05 DEBUG "metric": { 17:59:05 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:59:05 DEBUG }, 17:59:05 DEBUG "value": [ 17:59:05 DEBUG 17:59:05 DEBUG 17:59:05 DEBUG 17:59:05 DEBUG [...] 17:59:05 DEBUG 17:59:05 DEBUG 17:59:05 DEBUG 17:59:05 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:59:05 DEBUG }, 17:59:05 DEBUG "value": [ 17:59:05 DEBUG 1713545945, 17:59:05 DEBUG "2339.7666666666664" 17:59:05 DEBUG ] 17:59:05 DEBUG } 17:59:05 DEBUG ] 17:59:05 DEBUG } 17:59:05 DEBUG } 17:59:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545945" 17:59:05 INFO [http_cmd]: http status code OK 17:59:05 DEBUG --- status code --- 17:59:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:05 DEBUG --- http response --- 17:59:05 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 17:59:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713545946" 17:59:06 INFO [http_cmd]: http status code OK 17:59:06 DEBUG --- status code --- 17:59:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:06 DEBUG --- http response --- 17:59:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:59:06 DEBUG ----- output ----- 17:59:06 DEBUG { 17:59:06 DEBUG "status": "success", 17:59:06 DEBUG "data": { 17:59:06 DEBUG "resultType": "vector", 17:59:06 DEBUG "result": [ 17:59:06 DEBUG { 17:59:06 DEBUG "metric": { 17:59:06 DEBUG "pod": "ds-idrepo-0" 17:59:06 DEBUG }, 17:59:06 DEBUG "value": [ 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG [...] 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG "pod": "ds-cts-2" 17:59:06 DEBUG }, 17:59:06 DEBUG "value": [ 17:59:06 DEBUG 1713545946, 17:59:06 DEBUG "0" 17:59:06 DEBUG ] 17:59:06 DEBUG } 17:59:06 DEBUG ] 17:59:06 DEBUG } 17:59:06 DEBUG } 17:59:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545946" 17:59:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545946" 17:59:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545946" 17:59:06 INFO [http_cmd]: http status code OK 17:59:06 INFO [http_cmd]: http status code OK 17:59:06 DEBUG --- status code --- 17:59:06 DEBUG --- status code --- 17:59:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:06 DEBUG --- http response --- 17:59:06 DEBUG --- http response --- 17:59:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:59:06 INFO [http_cmd]: http status code OK 17:59:06 DEBUG ----- output ----- 17:59:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:59:06 DEBUG --- status code --- 17:59:06 DEBUG { 17:59:06 DEBUG ----- output ----- 17:59:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:06 DEBUG "status": "success", 17:59:06 DEBUG { 17:59:06 DEBUG --- http response --- 17:59:06 DEBUG "data": { 17:59:06 DEBUG "status": "success", 17:59:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:59:06 DEBUG "resultType": "vector", 17:59:06 DEBUG "data": { 17:59:06 DEBUG ----- output ----- 17:59:06 DEBUG "result": [ 17:59:06 DEBUG "resultType": "vector", 17:59:06 DEBUG { 17:59:06 DEBUG { 17:59:06 DEBUG "result": [ 17:59:06 DEBUG "status": "success", 17:59:06 DEBUG "metric": { 17:59:06 DEBUG { 17:59:06 DEBUG "data": { 17:59:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:59:06 DEBUG "metric": { 17:59:06 DEBUG "resultType": "vector", 17:59:06 DEBUG }, 17:59:06 DEBUG "pod": "ds-idrepo-0" 17:59:06 DEBUG "result": [ 17:59:06 DEBUG "value": [ 17:59:06 DEBUG }, 17:59:06 DEBUG { 17:59:06 DEBUG 17:59:06 DEBUG "value": [ 17:59:06 DEBUG "metric": { 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG }, 17:59:06 DEBUG "value": [ 17:59:06 DEBUG 17:59:06 DEBUG [...] 17:59:06 DEBUG 17:59:06 DEBUG [...] 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG [...] 17:59:06 DEBUG 17:59:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:59:06 DEBUG 17:59:06 DEBUG "pod": "ds-idrepo-2" 17:59:06 DEBUG }, 17:59:06 DEBUG 17:59:06 DEBUG }, 17:59:06 DEBUG "value": [ 17:59:06 DEBUG 17:59:06 DEBUG "value": [ 17:59:06 DEBUG 1713545946, 17:59:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:59:06 DEBUG 1713545946, 17:59:06 DEBUG "0" 17:59:06 DEBUG }, 17:59:06 DEBUG "16278.252554831768" 17:59:06 DEBUG ] 17:59:06 DEBUG "value": [ 17:59:06 DEBUG ] 17:59:06 DEBUG } 17:59:06 DEBUG 1713545946, 17:59:06 DEBUG } 17:59:06 DEBUG ] 17:59:06 DEBUG "0" 17:59:06 DEBUG ] 17:59:06 DEBUG } 17:59:06 DEBUG ] 17:59:06 DEBUG } 17:59:06 DEBUG } 17:59:06 DEBUG } 17:59:06 DEBUG } 17:59:06 DEBUG ] 17:59:06 DEBUG } 17:59:06 DEBUG } 17:59:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545946" 17:59:06 INFO [http_cmd]: http status code OK 17:59:06 DEBUG --- status code --- 17:59:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:06 DEBUG --- http response --- 17:59:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:59:06 DEBUG ----- output ----- 17:59:06 DEBUG { 17:59:06 DEBUG "status": "success", 17:59:06 DEBUG "data": { 17:59:06 DEBUG "resultType": "vector", 17:59:06 DEBUG "result": [ 17:59:06 DEBUG { 17:59:06 DEBUG "metric": { 17:59:06 DEBUG "instance": "10.142.0.107:9100" 17:59:06 DEBUG }, 17:59:06 DEBUG "value": [ 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG [...] 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG "instance": "10.142.0.43:9100" 17:59:06 DEBUG }, 17:59:06 DEBUG "value": [ 17:59:06 DEBUG 1713545946, 17:59:06 DEBUG "0.31299999999999195" 17:59:06 DEBUG ] 17:59:06 DEBUG } 17:59:06 DEBUG ] 17:59:06 DEBUG } 17:59:06 DEBUG } 17:59:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545946" 17:59:06 INFO [http_cmd]: http status code OK 17:59:06 DEBUG --- status code --- 17:59:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:06 DEBUG --- http response --- 17:59:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:59:06 DEBUG ----- output ----- 17:59:06 DEBUG { 17:59:06 DEBUG "status": "success", 17:59:06 DEBUG "data": { 17:59:06 DEBUG "resultType": "vector", 17:59:06 DEBUG "result": [ 17:59:06 DEBUG { 17:59:06 DEBUG "metric": { 17:59:06 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:59:06 DEBUG }, 17:59:06 DEBUG "value": [ 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG [...] 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG 17:59:06 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:59:06 DEBUG }, 17:59:06 DEBUG "value": [ 17:59:06 DEBUG 1713545946, 17:59:06 DEBUG "3142288.5" 17:59:06 DEBUG ] 17:59:06 DEBUG } 17:59:06 DEBUG ] 17:59:06 DEBUG } 17:59:06 DEBUG } 17:59:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545947" 17:59:07 INFO [http_cmd]: http status code OK 17:59:07 DEBUG --- status code --- 17:59:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:07 DEBUG --- http response --- 17:59:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:59:07 DEBUG ----- output ----- 17:59:07 DEBUG { 17:59:07 DEBUG "status": "success", 17:59:07 DEBUG "data": { 17:59:07 DEBUG "resultType": "vector", 17:59:07 DEBUG "result": [ 17:59:07 DEBUG { 17:59:07 DEBUG "metric": { 17:59:07 DEBUG "instance": "10.142.0.107:9100" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG [...] 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG "instance": "10.142.0.43:9100" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 1713545947, 17:59:07 DEBUG "1808718709760" 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG } 17:59:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545947" 17:59:07 INFO [http_cmd]: http status code OK 17:59:07 DEBUG --- status code --- 17:59:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:07 DEBUG --- http response --- 17:59:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:59:07 DEBUG ----- output ----- 17:59:07 DEBUG { 17:59:07 DEBUG "status": "success", 17:59:07 DEBUG "data": { 17:59:07 DEBUG "resultType": "vector", 17:59:07 DEBUG "result": [ 17:59:07 DEBUG { 17:59:07 DEBUG "metric": { 17:59:07 DEBUG "pod": "ds-cts-0" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG [...] 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG "pod": "ds-cts-2" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 1713545947, 17:59:07 DEBUG "0.00042794666666666665" 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG } 17:59:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545947" 17:59:07 INFO [http_cmd]: http status code OK 17:59:07 DEBUG --- status code --- 17:59:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:07 DEBUG --- http response --- 17:59:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:59:07 DEBUG ----- output ----- 17:59:07 DEBUG { 17:59:07 DEBUG "status": "success", 17:59:07 DEBUG "data": { 17:59:07 DEBUG "resultType": "vector", 17:59:07 DEBUG "result": [ 17:59:07 DEBUG { 17:59:07 DEBUG "metric": { 17:59:07 DEBUG "pod": "ds-idrepo-0" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG [...] 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG "pod": "ds-idrepo-2" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 1713545947, 17:59:07 DEBUG "0" 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG } 17:59:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713545947" 17:59:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545947" 17:59:07 INFO [http_cmd]: http status code OK 17:59:07 DEBUG --- status code --- 17:59:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:07 DEBUG --- http response --- 17:59:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:59:07 INFO [http_cmd]: http status code OK 17:59:07 DEBUG ----- output ----- 17:59:07 DEBUG --- status code --- 17:59:07 DEBUG { 17:59:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:07 DEBUG "status": "success", 17:59:07 DEBUG --- http response --- 17:59:07 DEBUG "data": { 17:59:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:59:07 DEBUG "resultType": "vector", 17:59:07 DEBUG ----- output ----- 17:59:07 DEBUG { 17:59:07 DEBUG "result": [ 17:59:07 DEBUG "status": "success", 17:59:07 DEBUG { 17:59:07 DEBUG "data": { 17:59:07 DEBUG "metric": { 17:59:07 DEBUG "resultType": "vector", 17:59:07 DEBUG "pod": "ds-cts-0" 17:59:07 DEBUG "result": [ 17:59:07 DEBUG }, 17:59:07 DEBUG { 17:59:07 DEBUG "value": [ 17:59:07 DEBUG "metric": { 17:59:07 DEBUG 17:59:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:59:07 DEBUG 17:59:07 DEBUG }, 17:59:07 DEBUG 17:59:07 DEBUG "value": [ 17:59:07 DEBUG [...] 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG [...] 17:59:07 DEBUG "pod": "ds-cts-2" 17:59:07 DEBUG 17:59:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545947" 17:59:07 DEBUG }, 17:59:07 DEBUG 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 17:59:07 DEBUG 1713545947, 17:59:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:59:07 DEBUG "7.933333333333334" 17:59:07 DEBUG }, 17:59:07 DEBUG ] 17:59:07 DEBUG "value": [ 17:59:07 DEBUG } 17:59:07 DEBUG 1713545947, 17:59:07 DEBUG ] 17:59:07 DEBUG "2533613568" 17:59:07 DEBUG } 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG } 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG } 17:59:07 INFO [http_cmd]: http status code OK 17:59:07 DEBUG --- status code --- 17:59:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:07 DEBUG --- http response --- 17:59:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:59:07 DEBUG ----- output ----- 17:59:07 DEBUG { 17:59:07 DEBUG "status": "success", 17:59:07 DEBUG "data": { 17:59:07 DEBUG "resultType": "vector", 17:59:07 DEBUG "result": [ 17:59:07 DEBUG { 17:59:07 DEBUG "metric": { 17:59:07 DEBUG "pod": "ds-idrepo-0" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG [...] 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG "pod": "ds-idrepo-2" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 1713545947, 17:59:07 DEBUG "0" 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG } 17:59:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545947" 17:59:07 INFO [http_cmd]: http status code OK 17:59:07 DEBUG --- status code --- 17:59:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:07 DEBUG --- http response --- 17:59:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:59:07 DEBUG ----- output ----- 17:59:07 DEBUG { 17:59:07 DEBUG "status": "success", 17:59:07 DEBUG "data": { 17:59:07 DEBUG "resultType": "vector", 17:59:07 DEBUG "result": [ 17:59:07 DEBUG { 17:59:07 DEBUG "metric": { 17:59:07 DEBUG "instance": "10.142.0.107:9100" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG [...] 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG "instance": "10.142.0.43:9100" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 1713545947, 17:59:07 DEBUG "16251416.633333333" 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG } 17:59:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545947" 17:59:07 INFO [http_cmd]: http status code OK 17:59:07 DEBUG --- status code --- 17:59:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:07 DEBUG --- http response --- 17:59:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:59:07 DEBUG ----- output ----- 17:59:07 DEBUG { 17:59:07 DEBUG "status": "success", 17:59:07 DEBUG "data": { 17:59:07 DEBUG "resultType": "vector", 17:59:07 DEBUG "result": [ 17:59:07 DEBUG { 17:59:07 DEBUG "metric": { 17:59:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG [...] 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG "pod": "ds-idrepo-2" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 1713545947, 17:59:07 DEBUG "5262.893683141098" 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG } 17:59:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545947" 17:59:07 INFO [http_cmd]: http status code OK 17:59:07 DEBUG --- status code --- 17:59:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:07 DEBUG --- http response --- 17:59:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:59:07 DEBUG ----- output ----- 17:59:07 DEBUG { 17:59:07 DEBUG "status": "success", 17:59:07 DEBUG "data": { 17:59:07 DEBUG "resultType": "vector", 17:59:07 DEBUG "result": [ 17:59:07 DEBUG { 17:59:07 DEBUG "metric": { 17:59:07 DEBUG "instance": "10.142.0.107:9100" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG [...] 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG 17:59:07 DEBUG "instance": "10.142.0.43:9100" 17:59:07 DEBUG }, 17:59:07 DEBUG "value": [ 17:59:07 DEBUG 1713545947, 17:59:07 DEBUG "16242393.6" 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG ] 17:59:07 DEBUG } 17:59:07 DEBUG } 17:59:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545948" 17:59:08 INFO [http_cmd]: http status code OK 17:59:08 DEBUG --- status code --- 17:59:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:08 DEBUG --- http response --- 17:59:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 17:59:08 DEBUG ----- output ----- 17:59:08 DEBUG { 17:59:08 DEBUG "status": "success", 17:59:08 DEBUG "data": { 17:59:08 DEBUG "resultType": "vector", 17:59:08 DEBUG "result": [ 17:59:08 DEBUG { 17:59:08 DEBUG "metric": { 17:59:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:59:08 DEBUG }, 17:59:08 DEBUG "value": [ 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG [...] 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG "pod": "lodemon-79cd4858c7-2r8x4" 17:59:08 DEBUG }, 17:59:08 DEBUG "value": [ 17:59:08 DEBUG 1713545948, 17:59:08 DEBUG "0" 17:59:08 DEBUG ] 17:59:08 DEBUG } 17:59:08 DEBUG ] 17:59:08 DEBUG } 17:59:08 DEBUG } 17:59:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545948" 17:59:08 INFO [http_cmd]: http status code OK 17:59:08 DEBUG --- status code --- 17:59:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:08 DEBUG --- http response --- 17:59:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:59:08 DEBUG ----- output ----- 17:59:08 DEBUG { 17:59:08 DEBUG "status": "success", 17:59:08 DEBUG "data": { 17:59:08 DEBUG "resultType": "vector", 17:59:08 DEBUG "result": [ 17:59:08 DEBUG { 17:59:08 DEBUG "metric": { 17:59:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:59:08 DEBUG }, 17:59:08 DEBUG "value": [ 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG [...] 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:59:08 DEBUG }, 17:59:08 DEBUG "value": [ 17:59:08 DEBUG 1713545948, 17:59:08 DEBUG "0" 17:59:08 DEBUG ] 17:59:08 DEBUG } 17:59:08 DEBUG ] 17:59:08 DEBUG } 17:59:08 DEBUG } 17:59:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713545948" 17:59:08 INFO [http_cmd]: http status code OK 17:59:08 DEBUG --- status code --- 17:59:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:08 DEBUG --- http response --- 17:59:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:59:08 DEBUG ----- output ----- 17:59:08 DEBUG { 17:59:08 DEBUG "status": "success", 17:59:08 DEBUG "data": { 17:59:08 DEBUG "resultType": "vector", 17:59:08 DEBUG "result": [ 17:59:08 DEBUG { 17:59:08 DEBUG "metric": { 17:59:08 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:59:08 DEBUG }, 17:59:08 DEBUG "value": [ 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG [...] 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:59:08 DEBUG }, 17:59:08 DEBUG "value": [ 17:59:08 DEBUG 1713545948, 17:59:08 DEBUG "0.027" 17:59:08 DEBUG ] 17:59:08 DEBUG } 17:59:08 DEBUG ] 17:59:08 DEBUG } 17:59:08 DEBUG } 17:59:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545948" 17:59:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713545948" 17:59:08 INFO [http_cmd]: http status code OK 17:59:08 DEBUG --- status code --- 17:59:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:08 DEBUG --- http response --- 17:59:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 17:59:08 DEBUG ----- output ----- 17:59:08 DEBUG { 17:59:08 DEBUG "status": "success", 17:59:08 DEBUG "data": { 17:59:08 DEBUG "resultType": "vector", 17:59:08 DEBUG "result": [ 17:59:08 DEBUG { 17:59:08 DEBUG "metric": { 17:59:08 DEBUG "pod": "ds-cts-0" 17:59:08 DEBUG }, 17:59:08 DEBUG "value": [ 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG [...] 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG "pod": "ds-cts-2" 17:59:08 DEBUG }, 17:59:08 DEBUG "value": [ 17:59:08 DEBUG 1713545948, 17:59:08 DEBUG "171.06666666666666" 17:59:08 DEBUG ] 17:59:08 DEBUG } 17:59:08 DEBUG ] 17:59:08 DEBUG } 17:59:08 DEBUG } 17:59:08 INFO [http_cmd]: http status code OK 17:59:08 DEBUG --- status code --- 17:59:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:08 DEBUG --- http response --- 17:59:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:59:08 DEBUG ----- output ----- 17:59:08 DEBUG { 17:59:08 DEBUG "status": "success", 17:59:08 DEBUG "data": { 17:59:08 DEBUG "resultType": "vector", 17:59:08 DEBUG "result": [ 17:59:08 DEBUG { 17:59:08 DEBUG "metric": { 17:59:08 DEBUG "instance": "10.142.0.107:9100" 17:59:08 DEBUG }, 17:59:08 DEBUG "value": [ 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG [...] 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG 17:59:08 DEBUG "instance": "10.142.0.43:9100" 17:59:08 DEBUG }, 17:59:08 DEBUG "value": [ 17:59:08 DEBUG 1713545948, 17:59:08 DEBUG "0.36646666666659655" 17:59:08 DEBUG ] 17:59:08 DEBUG } 17:59:08 DEBUG ] 17:59:08 DEBUG } 17:59:08 DEBUG } 17:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545949" 17:59:09 INFO [http_cmd]: http status code OK 17:59:09 DEBUG --- status code --- 17:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:09 DEBUG --- http response --- 17:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:59:09 DEBUG ----- output ----- 17:59:09 DEBUG { 17:59:09 DEBUG "status": "success", 17:59:09 DEBUG "data": { 17:59:09 DEBUG "resultType": "vector", 17:59:09 DEBUG "result": [ 17:59:09 DEBUG { 17:59:09 DEBUG "metric": { 17:59:09 DEBUG "pod": "am-6fd554f8f7-2pw5f" 17:59:09 DEBUG }, 17:59:09 DEBUG "value": [ 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG [...] 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG "pod": "am-6fd554f8f7-wxcsj" 17:59:09 DEBUG }, 17:59:09 DEBUG "value": [ 17:59:09 DEBUG 1713545949, 17:59:09 DEBUG "0" 17:59:09 DEBUG ] 17:59:09 DEBUG } 17:59:09 DEBUG ] 17:59:09 DEBUG } 17:59:09 DEBUG } 17:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545949" 17:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713545949" 17:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713545949" 17:59:09 INFO [http_cmd]: http status code OK 17:59:09 DEBUG --- status code --- 17:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:09 DEBUG --- http response --- 17:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 17:59:09 DEBUG ----- output ----- 17:59:09 DEBUG { 17:59:09 DEBUG "status": "success", 17:59:09 DEBUG "data": { 17:59:09 DEBUG "resultType": "vector", 17:59:09 DEBUG "result": [ 17:59:09 DEBUG { 17:59:09 DEBUG "metric": { 17:59:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 17:59:09 DEBUG }, 17:59:09 DEBUG "value": [ 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG [...] 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 INFO [http_cmd]: http status code OK 17:59:09 DEBUG "pod": "ds-idrepo-2" 17:59:09 DEBUG --- status code --- 17:59:09 DEBUG }, 17:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:09 DEBUG "value": [ 17:59:09 DEBUG --- http response --- 17:59:09 DEBUG 1713545949, 17:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 17:59:09 DEBUG "0" 17:59:09 DEBUG ----- output ----- 17:59:09 DEBUG ] 17:59:09 DEBUG { 17:59:09 DEBUG } 17:59:09 DEBUG "status": "success", 17:59:09 DEBUG ] 17:59:09 DEBUG "data": { 17:59:09 DEBUG } 17:59:09 DEBUG "resultType": "vector", 17:59:09 DEBUG } 17:59:09 DEBUG "result": [ 17:59:09 DEBUG { 17:59:09 DEBUG "metric": { 17:59:09 DEBUG "pod": "ds-idrepo-0" 17:59:09 DEBUG }, 17:59:09 DEBUG "value": [ 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG [...] 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG "pod": "ds-idrepo-2" 17:59:09 DEBUG }, 17:59:09 DEBUG "value": [ 17:59:09 DEBUG 1713545949, 17:59:09 DEBUG "0" 17:59:09 DEBUG ] 17:59:09 DEBUG } 17:59:09 DEBUG ] 17:59:09 DEBUG } 17:59:09 DEBUG } 17:59:09 INFO [http_cmd]: http status code OK 17:59:09 DEBUG --- status code --- 17:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 17:59:09 DEBUG --- http response --- 17:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 17:59:09 DEBUG ----- output ----- 17:59:09 DEBUG { 17:59:09 DEBUG "status": "success", 17:59:09 DEBUG "data": { 17:59:09 DEBUG "resultType": "vector", 17:59:09 DEBUG "result": [ 17:59:09 DEBUG { 17:59:09 DEBUG "metric": { 17:59:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 17:59:09 DEBUG }, 17:59:09 DEBUG "value": [ 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG [...] 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG 17:59:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 17:59:09 DEBUG }, 17:59:09 DEBUG "value": [ 17:59:09 DEBUG 1713545949, 17:59:09 DEBUG "1.1118912518717992" 17:59:09 DEBUG ] 17:59:09 DEBUG } 17:59:09 DEBUG ] 17:59:09 DEBUG } 17:59:09 DEBUG } 17:59:09 INFO 17:59:09 INFO [loop_until]: kubectl --namespace=xlou top pods 17:59:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:59:10 INFO [loop_until]: OK (rc = 0) 17:59:10 DEBUG --- stdout --- 17:59:10 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-6fd554f8f7-2pw5f 2142m 8904Mi am-6fd554f8f7-btd4s 2117m 8905Mi am-6fd554f8f7-wxcsj 2065m 8923Mi ds-cts-0 8943m 13834Mi ds-cts-1 9507m 13191Mi ds-cts-2 9673m 12948Mi ds-idrepo-0 650m 3451Mi ds-idrepo-1 639m 3411Mi ds-idrepo-2 640m 3472Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1173Mi idm-b85f9d978-wnhxb 4m 1217Mi lodemon-79cd4858c7-2r8x4 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1034m 2731Mi 17:59:10 DEBUG --- stderr --- 17:59:10 DEBUG 17:59:10 INFO 17:59:10 INFO [loop_until]: kubectl --namespace=xlou top node 17:59:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 17:59:10 INFO [loop_until]: OK (rc = 0) 17:59:10 DEBUG --- stdout --- 17:59:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6171Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 747m 2% 7021Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 46m 0% 2692Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 2225m 7% 12214Mi 10% gke-xlou-cdm-default-pool-1b0769c8-w15q 8606m 28% 16877Mi 15% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5240Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2183m 7% 11820Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 75m 0% 3210Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 2329m 7% 11165Mi 10% gke-xlou-cdm-ds-0f601751-j488 710m 2% 6699Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 9695m 32% 18199Mi 16% gke-xlou-cdm-ds-0f601751-nj6q 735m 2% 7252Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 10181m 34% 15320Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 1085m 6% 6685Mi 11% 17:59:10 DEBUG --- stderr --- 17:59:10 DEBUG 17:59:16 INFO Finished: True 17:59:16 INFO Waiting for threads to register finish flag 18:00:10 INFO Done. Have a nice day! :) 127.0.0.1 - - [19/Apr/2024 18:00:10] "GET /monitoring/stop HTTP/1.1" 200 - 18:00:13 WARNING File /tmp/lodemon_data-20240419_165801/node_disk_read_bytes_total.json does not exist. Skipping graph generation... WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p6521', ...) 127.0.0.1 - - [19/Apr/2024 18:00:17] "GET /monitoring/process HTTP/1.1" 200 -