==================================================================================================== ========================================= Pod describe ========================================= ==================================================================================================== Name: lodemon-6f4955bbbb-clfwf Namespace: xlou Priority: 0 Service Account: k8s-svc-acct-lodemon Node: gke-xlou-cdm-ds-0f601751-mc5d/10.142.0.43 Start Time: Sat, 20 Apr 2024 18:18:35 +0000 Labels: app=lodemon app.kubernetes.io/name=lodemon pod-template-hash=6f4955bbbb skaffold.dev/run-id=b90eda45-3bfd-48d2-97ef-56c44f04b7b0 Annotations: Status: Running IP: 10.106.46.172 IPs: IP: 10.106.46.172 Controlled By: ReplicaSet/lodemon-6f4955bbbb Containers: lodemon: Container ID: containerd://a7d6aa67c49ee3b85ab0494c0b001fb738f97cec656ccc1394198f0bb141bdcd Image: gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable Image ID: gcr.io/engineeringpit/lodestar-images/lodestarbox@sha256:03f76e6326d713f4d73dc474e0e25fbbbaecc7c1d8c0590bfd94d68ac0d98c97 Port: 8080/TCP Host Port: 0/TCP Command: python3 Args: /lodestar/scripts/lodemon_run.py -W default State: Running Started: Sat, 20 Apr 2024 18:18:36 +0000 Ready: True Restart Count: 0 Limits: cpu: 2 memory: 2Gi Requests: cpu: 1 memory: 1Gi Liveness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Readiness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Environment: SKAFFOLD_PROFILE: medium Mounts: /lodestar/config/config.yaml from config (rw,path="config.yaml") /lodestar/config/deployments_to_monitor.yaml from deployments-config (rw,path="deployments_to_monitor.yaml") /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-5jscr (ro) Conditions: Type Status Initialized True Ready True ContainersReady True PodScheduled True Volumes: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-config Optional: false deployments-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-deployments Optional: false kube-api-access-5jscr: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: ==================================================================================================== =========================================== Pod logs =========================================== ==================================================================================================== * Serving Flask app 'lodemon_run' * Debug mode: off WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. * Running on all addresses (0.0.0.0) * Running on http://127.0.0.1:8080 * Running on http://10.106.46.172:8080 Press CTRL+C to quit 19:19:43 INFO 19:19:43 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:43 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:43 INFO [loop_until]: OK (rc = 0) 19:19:43 DEBUG --- stdout --- 19:19:43 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:43 DEBUG --- stderr --- 19:19:43 DEBUG 19:19:43 INFO 19:19:43 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:43 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:43 INFO [loop_until]: OK (rc = 0) 19:19:43 DEBUG --- stdout --- 19:19:43 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:43 DEBUG --- stderr --- 19:19:43 DEBUG 19:19:43 INFO 19:19:43 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:43 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:44 INFO [loop_until]: OK (rc = 0) 19:19:44 DEBUG --- stdout --- 19:19:44 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:44 DEBUG --- stderr --- 19:19:44 DEBUG 19:19:44 INFO 19:19:44 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:44 INFO [loop_until]: OK (rc = 0) 19:19:44 DEBUG --- stdout --- 19:19:44 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:44 DEBUG --- stderr --- 19:19:44 DEBUG 19:19:44 INFO 19:19:44 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:44 INFO [loop_until]: OK (rc = 0) 19:19:44 DEBUG --- stdout --- 19:19:44 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:44 DEBUG --- stderr --- 19:19:44 DEBUG 19:19:44 INFO 19:19:44 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:44 INFO [loop_until]: OK (rc = 0) 19:19:44 DEBUG --- stdout --- 19:19:44 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:44 DEBUG --- stderr --- 19:19:44 DEBUG 19:19:44 INFO 19:19:44 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:44 INFO [loop_until]: OK (rc = 0) 19:19:44 DEBUG --- stdout --- 19:19:44 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:44 DEBUG --- stderr --- 19:19:44 DEBUG 19:19:45 INFO 19:19:45 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:45 INFO [loop_until]: OK (rc = 0) 19:19:45 DEBUG --- stdout --- 19:19:45 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:45 DEBUG --- stderr --- 19:19:45 DEBUG 19:19:45 INFO 19:19:45 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:45 INFO [loop_until]: OK (rc = 0) 19:19:45 DEBUG --- stdout --- 19:19:45 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:45 DEBUG --- stderr --- 19:19:45 DEBUG 19:19:45 INFO 19:19:45 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:45 INFO [loop_until]: OK (rc = 0) 19:19:45 DEBUG --- stdout --- 19:19:45 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:45 DEBUG --- stderr --- 19:19:45 DEBUG 19:19:45 INFO 19:19:45 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:45 INFO [loop_until]: OK (rc = 0) 19:19:45 DEBUG --- stdout --- 19:19:45 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:45 DEBUG --- stderr --- 19:19:45 DEBUG 19:19:45 INFO 19:19:45 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:46 INFO [loop_until]: OK (rc = 0) 19:19:46 DEBUG --- stdout --- 19:19:46 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:46 DEBUG --- stderr --- 19:19:46 DEBUG 19:19:46 INFO 19:19:46 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:46 INFO [loop_until]: OK (rc = 0) 19:19:46 DEBUG --- stdout --- 19:19:46 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:46 DEBUG --- stderr --- 19:19:46 DEBUG 19:19:46 INFO 19:19:46 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:46 INFO [loop_until]: OK (rc = 0) 19:19:46 DEBUG --- stdout --- 19:19:46 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:46 DEBUG --- stderr --- 19:19:46 DEBUG 19:19:46 INFO 19:19:46 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:46 INFO [loop_until]: OK (rc = 0) 19:19:46 DEBUG --- stdout --- 19:19:46 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:46 DEBUG --- stderr --- 19:19:46 DEBUG 19:19:46 INFO 19:19:46 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:46 INFO [loop_until]: OK (rc = 0) 19:19:46 DEBUG --- stdout --- 19:19:46 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:46 DEBUG --- stderr --- 19:19:46 DEBUG 19:19:46 INFO 19:19:46 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:47 INFO [loop_until]: OK (rc = 0) 19:19:47 DEBUG --- stdout --- 19:19:47 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:47 DEBUG --- stderr --- 19:19:47 DEBUG 19:19:47 INFO 19:19:47 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:47 INFO [loop_until]: OK (rc = 0) 19:19:47 DEBUG --- stdout --- 19:19:47 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:47 DEBUG --- stderr --- 19:19:47 DEBUG 19:19:47 INFO 19:19:47 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:47 INFO [loop_until]: OK (rc = 0) 19:19:47 DEBUG --- stdout --- 19:19:47 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:47 DEBUG --- stderr --- 19:19:47 DEBUG 19:19:47 INFO 19:19:47 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:47 INFO [loop_until]: OK (rc = 0) 19:19:47 DEBUG --- stdout --- 19:19:47 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:47 DEBUG --- stderr --- 19:19:47 DEBUG 19:19:47 INFO 19:19:47 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:47 INFO [loop_until]: OK (rc = 0) 19:19:47 DEBUG --- stdout --- 19:19:47 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:47 DEBUG --- stderr --- 19:19:47 DEBUG 19:19:48 INFO 19:19:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:48 INFO [loop_until]: OK (rc = 0) 19:19:48 DEBUG --- stdout --- 19:19:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:48 DEBUG --- stderr --- 19:19:48 DEBUG 19:19:48 INFO 19:19:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:48 INFO [loop_until]: OK (rc = 0) 19:19:48 DEBUG --- stdout --- 19:19:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:48 DEBUG --- stderr --- 19:19:48 DEBUG 19:19:48 INFO 19:19:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:48 INFO [loop_until]: OK (rc = 0) 19:19:48 DEBUG --- stdout --- 19:19:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:48 DEBUG --- stderr --- 19:19:48 DEBUG 19:19:48 INFO 19:19:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:48 INFO [loop_until]: OK (rc = 0) 19:19:48 DEBUG --- stdout --- 19:19:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:48 DEBUG --- stderr --- 19:19:48 DEBUG 19:19:48 INFO 19:19:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:49 INFO [loop_until]: OK (rc = 0) 19:19:49 DEBUG --- stdout --- 19:19:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:49 DEBUG --- stderr --- 19:19:49 DEBUG 19:19:49 INFO 19:19:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 19:19:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:49 INFO [loop_until]: OK (rc = 0) 19:19:49 DEBUG --- stdout --- 19:19:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 19:19:49 DEBUG --- stderr --- 19:19:49 DEBUG 19:19:49 INFO Initializing monitoring instance threads 19:19:49 DEBUG Monitoring instance thread list: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 19:19:49 INFO Starting instance threads 19:19:49 INFO 19:19:49 INFO [loop_until]: kubectl --namespace=xlou top node 19:19:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:49 INFO Thread started 19:19:49 INFO 19:19:49 INFO [loop_until]: kubectl --namespace=xlou top pods 19:19:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:19:49 INFO Thread started 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG --- http response --- 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG --- status code --- 19:19:49 INFO Thread started 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:19:49 DEBUG --- status code --- 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 DEBUG ----- output ----- 19:19:49 INFO Thread started 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG { 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG { 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG "data": { 19:19:49 DEBUG --- status code --- 19:19:49 INFO Thread started 19:19:49 DEBUG --- http response --- 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:19:49 DEBUG --- status code --- 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 INFO Thread started 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG "data": { 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG "result": [ 19:19:49 DEBUG { 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "result": [ 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG "metric": { 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:49 DEBUG ----- output ----- 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG --- status code --- 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG { 19:19:49 DEBUG { 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:19:49 DEBUG { 19:19:49 DEBUG { 19:19:49 DEBUG ----- output ----- 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG { 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG "metric": { 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:49 DEBUG }, 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG "data": { 19:19:49 DEBUG "status": "success", 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 INFO [loop_until]: OK (rc = 0) 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG "data": { 19:19:49 DEBUG "data": { 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 DEBUG ----- output ----- 19:19:49 INFO Thread started 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:19:49 DEBUG { 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "data": { 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG --- stdout --- 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "result": [ 19:19:49 DEBUG { 19:19:49 DEBUG "metric": { 19:19:49 INFO Thread started 19:19:49 DEBUG }, 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG { 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG "data": { 19:19:49 DEBUG "result": [ 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG "result": [ 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17349m 8612Mi am-767f6b67b5-qcqz2 17239m 8703Mi am-767f6b67b5-vbltz 19486m 8665Mi ds-cts-0 18644m 10183Mi ds-cts-1 17721m 10086Mi ds-cts-2 19212m 10174Mi ds-idrepo-0 7355m 3889Mi ds-idrepo-1 7602m 3908Mi ds-idrepo-2 7454m 3753Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-6f4955bbbb-clfwf 2m 54Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9078m 665Mi 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG "result": [ 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG { 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 DEBUG "value": [ 19:19:49 INFO Thread started 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG { 19:19:49 INFO [loop_until]: OK (rc = 0) 19:19:49 DEBUG --- stdout --- 19:19:49 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6287Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7568m 25% 7626Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17032m 57% 11332Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3364Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18494m 61% 13468Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5202Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17184m 57% 11593Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14634m 49% 2975Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19503m 65% 10884Mi 9% gke-xlou-cdm-ds-0f601751-j488 8211m 27% 7148Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17678m 59% 15223Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8165m 27% 7866Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19340m 64% 12924Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9149m 57% 6446Mi 10% 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG { 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG --- stderr --- 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:19:49 DEBUG { 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:49 DEBUG }, 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG "data": { 19:19:49 INFO Thread started 19:19:49 DEBUG "result": [ 19:19:49 DEBUG "metric": { 19:19:49 DEBUG --- stderr --- 19:19:49 DEBUG 19:19:49 DEBUG { 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG "metric": { 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG { 19:19:49 DEBUG { 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG "value": [ 19:19:49 DEBUG [...] 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "result": [ 19:19:49 DEBUG { 19:19:49 INFO Thread started 19:19:49 DEBUG { 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:49 DEBUG "instance": "10.142.0.107:9100" 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG "metric": { 19:19:49 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG { 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:19:49 DEBUG }, 19:19:49 DEBUG "value": [ 19:19:49 DEBUG "metric": { 19:19:49 INFO Thread started 19:19:49 DEBUG [...] 19:19:49 DEBUG "data": { 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG }, 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG "data": { 19:19:49 DEBUG "data": { 19:19:49 DEBUG "metric": { 19:19:49 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:19:49 DEBUG 19:19:49 DEBUG --- http response --- 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG { 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG { 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG "data": { 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "instance": "10.142.0.107:9100" 19:19:49 DEBUG }, 19:19:49 DEBUG 19:19:49 DEBUG [...] 19:19:49 DEBUG 19:19:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:49 DEBUG }, 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637189" 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 INFO Thread started 19:19:49 DEBUG "result": [ 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG { 19:19:49 DEBUG 19:19:49 DEBUG "data": { 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG "data": { 19:19:49 DEBUG }, 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:19:49 DEBUG "result": [ 19:19:49 DEBUG "result": [ 19:19:49 DEBUG }, 19:19:49 DEBUG "value": [ 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG [...] 19:19:49 DEBUG "value": [ 19:19:49 DEBUG --- status code --- 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637189" 19:19:49 DEBUG { 19:19:49 INFO Thread started 19:19:49 DEBUG "data": { 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "result": [ 19:19:49 DEBUG "resultType": "vector", 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG "result": [ 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG { 19:19:49 DEBUG "metric": { 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG 19:19:49 DEBUG { 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG "metric": { 19:19:49 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:19:49 INFO Thread started 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "data": { 19:19:49 DEBUG [...] 19:19:49 DEBUG --- status code --- 19:19:49 ERROR [http_cmd]: ERROR 19:19:49 ERROR --- status code --- 19:19:49 DEBUG "result": [ 19:19:49 DEBUG { 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG { 19:19:49 DEBUG { 19:19:49 DEBUG "pod": "ds-idrepo-0" 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG [...] 19:19:49 DEBUG 19:19:49 DEBUG "metric": { 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG --- http response --- 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637189" 19:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637189" 19:19:49 INFO Thread started 19:19:49 DEBUG "result": [ 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:19:49 DEBUG { 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG "metric": { 19:19:49 DEBUG [...] 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:49 DEBUG "metric": { 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG }, 19:19:49 DEBUG { 19:19:49 DEBUG 19:19:49 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:19:49 DEBUG "0" 19:19:49 DEBUG 19:19:49 DEBUG "instance": "10.142.0.107:9100" 19:19:49 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:19:49 DEBUG [...] 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG }, 19:19:49 INFO All threads has been started 127.0.0.1 - - [20/Apr/2024 19:19:49] "GET /monitoring/start HTTP/1.1" 200 - 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:19:49 DEBUG { 19:19:49 DEBUG "result": [ 19:19:49 DEBUG 19:19:49 DEBUG --- http response --- 19:19:49 ERROR --- http response --- 19:19:49 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:49 DEBUG "data": { 19:19:49 DEBUG "value": [ 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG 19:19:49 DEBUG }, 19:19:49 DEBUG ] 19:19:49 DEBUG 19:19:49 DEBUG }, 19:19:49 DEBUG }, 19:19:49 DEBUG 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:49 DEBUG "value": [ 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG ----- output ----- 19:19:49 INFO [http_cmd]: http status code OK 19:19:49 DEBUG "metric": { 19:19:49 DEBUG { 19:19:49 DEBUG 19:19:49 DEBUG "instance": "10.142.0.43:9100" 19:19:49 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:19:49 DEBUG "metric": { 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG }, 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG 19:19:49 DEBUG { 19:19:49 DEBUG }, 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG 19:19:49 DEBUG "data": { 19:19:49 DEBUG 19:19:49 DEBUG "value": [ 19:19:49 DEBUG } 19:19:49 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:19:49 DEBUG "value": [ 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG { 19:19:49 DEBUG 19:19:49 DEBUG { 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG "pod": "ds-cts-0" 19:19:49 DEBUG "metric": { 19:19:49 DEBUG }, 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:19:49 DEBUG "pod": "ds-idrepo-0" 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:19:49 DEBUG "value": [ 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG "data": { 19:19:49 DEBUG "result": [ 19:19:49 DEBUG 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "instance": "10.142.0.43:9100" 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG ] 19:19:49 DEBUG }, 19:19:49 DEBUG 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG 19:19:49 DEBUG --- status code --- 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG }, 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG { 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "value": [ 19:19:49 DEBUG { 19:19:49 DEBUG 19:19:49 DEBUG "result": [ 19:19:49 DEBUG }, 19:19:49 DEBUG "0" 19:19:49 DEBUG } 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG "12989.412416386449" 19:19:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:19:49 DEBUG "data": { 19:19:49 DEBUG 19:19:49 DEBUG "data": { 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG "value": [ 19:19:49 DEBUG "80660493.33333334" 19:19:49 DEBUG "pod": "ds-idrepo-0" 19:19:49 DEBUG }, 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG }, 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG "metric": { 19:19:49 DEBUG [...] 19:19:49 DEBUG 19:19:49 DEBUG "value": [ 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG "8.90377554497318" 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG --- http response --- 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG [...] 19:19:49 DEBUG 19:19:49 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:19:49 DEBUG 19:19:49 DEBUG ] 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG { 19:19:49 DEBUG 19:19:49 DEBUG }, 19:19:49 DEBUG [...] 19:19:49 DEBUG 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG "result": [ 19:19:49 DEBUG 19:19:49 DEBUG "pod": "ds-cts-0" 19:19:49 DEBUG 19:19:49 DEBUG { 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:49 DEBUG "result": [ 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG "data": { 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG 19:19:49 DEBUG [...] 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:49 DEBUG }, 19:19:49 DEBUG 19:19:49 DEBUG [...] 19:19:49 DEBUG 19:19:49 DEBUG "pod": "ds-idrepo-2" 19:19:49 DEBUG }, 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG [...] 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG { 19:19:49 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:19:49 DEBUG }, 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG } 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "result": [ 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG "795746304" 19:19:49 DEBUG "value": [ 19:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:19:49 DEBUG ----- output ----- 19:19:49 DEBUG }, 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG "80655361.76666667" 19:19:49 DEBUG } 19:19:49 DEBUG { 19:19:49 DEBUG "metric": { 19:19:49 DEBUG } 19:19:49 DEBUG }, 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG "metric": { 19:19:49 DEBUG { 19:19:49 DEBUG "result": [ 19:19:49 DEBUG ] 19:19:49 DEBUG { 19:19:49 DEBUG "data": { 19:19:49 DEBUG 19:19:49 DEBUG "pod": "ds-idrepo-2" 19:19:49 DEBUG ] 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG { 19:19:49 DEBUG 19:19:49 DEBUG "value": [ 19:19:49 DEBUG "0" 19:19:49 DEBUG "metric": { 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:49 DEBUG } 19:19:49 DEBUG "value": [ 19:19:49 DEBUG "value": [ 19:19:49 DEBUG "instance": "10.142.0.43:9100" 19:19:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:49 DEBUG [...] 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG } 19:19:49 DEBUG { 19:19:49 DEBUG "metric": { 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG }, 19:19:49 DEBUG "value": [ 19:19:49 DEBUG } 19:19:49 DEBUG "2676" 19:19:49 DEBUG "status": "success", 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG ] 19:19:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:49 DEBUG } 19:19:49 DEBUG }, 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG }, 19:19:49 DEBUG }, 19:19:49 DEBUG 19:19:49 DEBUG "data": { 19:19:49 DEBUG } 19:19:49 DEBUG "metric": { 19:19:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:19:49 DEBUG "result": [ 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG ] 19:19:49 DEBUG ] 19:19:49 DEBUG "data": { 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG }, 19:19:49 DEBUG ] 19:19:49 DEBUG "value": [ 19:19:49 DEBUG "2675" 19:19:49 DEBUG "5.03077160231514" 19:19:49 DEBUG "value": [ 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "pod": "ds-cts-0" 19:19:49 DEBUG }, 19:19:49 DEBUG { 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG "resultType": "vector", 19:19:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:49 DEBUG 19:19:49 DEBUG ] 19:19:49 DEBUG "value": [ 19:19:49 DEBUG } 19:19:49 DEBUG 19:19:49 DEBUG ] 19:19:49 DEBUG ] 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG "result": [ 19:19:49 DEBUG }, 19:19:49 DEBUG "value": [ 19:19:49 DEBUG "metric": { 19:19:49 DEBUG "0" 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG "result": [ 19:19:49 DEBUG }, 19:19:49 DEBUG [...] 19:19:49 DEBUG } 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG "0.09200000000000916" 19:19:49 DEBUG 19:19:49 DEBUG "pod": "ds-cts-2" 19:19:49 DEBUG { 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG "pod": "ds-idrepo-0" 19:19:49 DEBUG ] 19:19:49 DEBUG [...] 19:19:49 DEBUG ] 19:19:49 DEBUG { 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG ] 19:19:49 DEBUG ] 19:19:49 DEBUG ] 19:19:49 DEBUG 19:19:49 DEBUG }, 19:19:49 DEBUG "metric": { 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG }, 19:19:49 DEBUG } 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG "pod": "ds-cts-2" 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG "value": [ 19:19:49 DEBUG "instance": "10.142.0.107:9100" 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG "value": [ 19:19:49 DEBUG ] 19:19:49 DEBUG 19:19:49 DEBUG "metric": { 19:19:49 DEBUG "2673.266666666667" 19:19:49 DEBUG }, 19:19:49 DEBUG [...] 19:19:49 DEBUG [...] 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG [...] 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG }, 19:19:49 DEBUG 19:19:49 DEBUG [...] 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG 19:19:49 DEBUG "instance": "10.142.0.107:9100" 19:19:49 DEBUG ] 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG 19:19:49 DEBUG "0" 19:19:49 DEBUG "value": [ 19:19:49 DEBUG [...] 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG "pod": "ds-idrepo-2" 19:19:49 DEBUG }, 19:19:49 DEBUG } 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG 19:19:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:49 DEBUG 19:19:49 DEBUG ] 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG }, 19:19:49 DEBUG "value": [ 19:19:49 DEBUG ] 19:19:49 DEBUG "33.666666666666664" 19:19:49 DEBUG }, 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:49 DEBUG }, 19:19:49 DEBUG 19:19:49 DEBUG [...] 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG "value": [ 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG 19:19:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:49 DEBUG }, 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG } 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG "0" 19:19:49 DEBUG 19:19:49 DEBUG ] 19:19:49 DEBUG "0" 19:19:49 DEBUG } 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG "480400" 19:19:49 DEBUG [...] 19:19:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG ] 19:19:49 DEBUG [...] 19:19:49 DEBUG 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG 19:19:49 DEBUG }, 19:19:49 DEBUG "pod": "ds-cts-2" 19:19:49 DEBUG "pod": "ds-cts-2" 19:19:49 DEBUG }, 19:19:49 DEBUG 19:19:49 DEBUG 19:19:49 DEBUG ] 19:19:49 DEBUG "2674.4666666666667" 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG "value": [ 19:19:49 DEBUG }, 19:19:49 DEBUG "value": [ 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG "instance": "10.142.0.43:9100" 19:19:49 DEBUG ] 19:19:49 DEBUG 19:19:49 DEBUG "value": [ 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG }, 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG } 19:19:49 DEBUG 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG "0.00043333333333333337" 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG "value": [ 19:19:49 DEBUG "0.152" 19:19:49 DEBUG ] 19:19:49 DEBUG "instance": "10.142.0.43:9100" 19:19:49 DEBUG "0" 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG }, 19:19:49 DEBUG ] 19:19:49 DEBUG ] 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG "value": [ 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG "3484062682112" 19:19:49 DEBUG ] 19:19:49 DEBUG 1713637189, 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG "0.2510000000000142" 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG ] 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG } 19:19:49 DEBUG } 19:20:49 INFO 19:20:49 INFO [loop_until]: kubectl --namespace=xlou top node 19:20:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:20:49 INFO 19:20:49 INFO [loop_until]: kubectl --namespace=xlou top pods 19:20:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 ERROR [http_cmd]: ERROR 19:20:49 ERROR --- status code --- 19:20:49 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:20:49 ERROR --- http response --- 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:20:49 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637249" 19:20:49 INFO [loop_until]: OK (rc = 0) 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637249" 19:20:49 DEBUG --- stdout --- 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [loop_until]: OK (rc = 0) 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17299m 8610Mi am-767f6b67b5-qcqz2 16976m 8703Mi am-767f6b67b5-vbltz 18589m 8665Mi ds-cts-0 18736m 10187Mi ds-cts-1 17417m 10100Mi ds-cts-2 18889m 10178Mi ds-idrepo-0 7208m 3889Mi ds-idrepo-1 7556m 3907Mi ds-idrepo-2 7365m 3753Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1239Mi lodemon-6f4955bbbb-clfwf 260m 61Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8761m 668Mi 19:20:49 DEBUG --- status code --- 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637249" 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG --- stdout --- 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG --- stderr --- 19:20:49 DEBUG 19:20:49 DEBUG --- status code --- 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 DEBUG --- status code --- 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG --- status code --- 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG --- status code --- 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637249" 19:20:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637249" 19:20:49 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6221Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7714m 25% 7635Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17017m 56% 11329Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18622m 62% 13477Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5213Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17215m 57% 11590Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14589m 48% 2954Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19443m 65% 10885Mi 9% gke-xlou-cdm-ds-0f601751-j488 8244m 27% 7147Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17344m 58% 15244Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8157m 27% 7866Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18811m 63% 12927Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9079m 57% 6449Mi 10% 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG --- status code --- 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG --- status code --- 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG --- status code --- 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG --- stderr --- 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG --- http response --- 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG --- status code --- 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 INFO [http_cmd]: http status code OK 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG --- status code --- 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:49 DEBUG { 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG { 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG { 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:20:49 DEBUG { 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG { 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG --- http response --- 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:20:49 DEBUG { 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG { 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG { 19:20:49 DEBUG { 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG { 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG { 19:20:49 DEBUG "data": { 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:20:49 DEBUG { 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:20:49 DEBUG { 19:20:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:20:49 DEBUG { 19:20:49 DEBUG "data": { 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG "data": { 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG { 19:20:49 DEBUG { 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "data": { 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "data": { 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG { 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG ----- output ----- 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG { 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG { 19:20:49 DEBUG "data": { 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "data": { 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG { 19:20:49 DEBUG "data": { 19:20:49 DEBUG "data": { 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "data": { 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "data": { 19:20:49 DEBUG "result": [ 19:20:49 DEBUG { 19:20:49 DEBUG { 19:20:49 DEBUG "data": { 19:20:49 DEBUG { 19:20:49 DEBUG { 19:20:49 DEBUG "data": { 19:20:49 DEBUG { 19:20:49 DEBUG "data": { 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "data": { 19:20:49 DEBUG "data": { 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "data": { 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG { 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "data": { 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "status": "success", 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG { 19:20:49 DEBUG "data": { 19:20:49 DEBUG { 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "result": [ 19:20:49 DEBUG { 19:20:49 DEBUG "data": { 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "result": [ 19:20:49 DEBUG { 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "data": { 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "data": { 19:20:49 DEBUG "data": { 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "data": { 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "data": { 19:20:49 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:20:49 DEBUG { 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "result": [ 19:20:49 DEBUG { 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG { 19:20:49 DEBUG { 19:20:49 DEBUG "metric": { 19:20:49 DEBUG { 19:20:49 DEBUG "result": [ 19:20:49 DEBUG { 19:20:49 DEBUG "instance": "10.142.0.107:9100" 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG { 19:20:49 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "resultType": "vector", 19:20:49 DEBUG }, 19:20:49 DEBUG "metric": { 19:20:49 DEBUG { 19:20:49 DEBUG { 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:20:49 DEBUG "metric": { 19:20:49 DEBUG { 19:20:49 DEBUG "metric": { 19:20:49 DEBUG }, 19:20:49 DEBUG "result": [ 19:20:49 DEBUG { 19:20:49 DEBUG { 19:20:49 DEBUG { 19:20:49 DEBUG { 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "metric": { 19:20:49 DEBUG }, 19:20:49 DEBUG { 19:20:49 DEBUG "result": [ 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "instance": "10.142.0.107:9100" 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:49 DEBUG }, 19:20:49 DEBUG { 19:20:49 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:20:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:49 DEBUG }, 19:20:49 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "pod": "ds-idrepo-0" 19:20:49 DEBUG "value": [ 19:20:49 DEBUG { 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "metric": { 19:20:49 DEBUG { 19:20:49 DEBUG { 19:20:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "metric": { 19:20:49 DEBUG { 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG "pod": "ds-idrepo-0" 19:20:49 DEBUG "instance": "10.142.0.107:9100" 19:20:49 DEBUG }, 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "metric": { 19:20:49 DEBUG }, 19:20:49 DEBUG }, 19:20:49 DEBUG "value": [ 19:20:49 DEBUG }, 19:20:49 DEBUG "pod": "ds-idrepo-0" 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:49 DEBUG "instance": "10.142.0.107:9100" 19:20:49 DEBUG "metric": { 19:20:49 DEBUG "metric": { 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:20:49 DEBUG "metric": { 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG }, 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG "pod": "ds-cts-0" 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG }, 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG "pod": "ds-cts-0" 19:20:49 DEBUG }, 19:20:49 DEBUG }, 19:20:49 DEBUG }, 19:20:49 DEBUG }, 19:20:49 DEBUG "pod": "ds-cts-0" 19:20:49 DEBUG "instance": "10.142.0.107:9100" 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG "pod": "ds-idrepo-0" 19:20:49 DEBUG "value": [ 19:20:49 DEBUG [...] 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "value": [ 19:20:49 DEBUG }, 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG "instance": "10.142.0.43:9100" 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:20:49 DEBUG [...] 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:49 DEBUG [...] 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG [...] 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG }, 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG "8.872149925671257" 19:20:49 DEBUG ] 19:20:49 DEBUG 19:20:49 DEBUG "80670257.2" 19:20:49 DEBUG 19:20:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG "pod": "ds-cts-2" 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:49 DEBUG "instance": "10.142.0.43:9100" 19:20:49 DEBUG }, 19:20:49 DEBUG "instance": "10.142.0.43:9100" 19:20:49 DEBUG "pod": "ds-idrepo-2" 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG 19:20:49 DEBUG "799846400" 19:20:49 DEBUG ] 19:20:49 DEBUG [...] 19:20:49 DEBUG } 19:20:49 DEBUG 19:20:49 DEBUG "pod": "ds-idrepo-2" 19:20:49 DEBUG 19:20:49 DEBUG ] 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG "value": [ 19:20:49 DEBUG }, 19:20:49 DEBUG }, 19:20:49 DEBUG "1266.7879731123498" 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:20:49 DEBUG "2672.5666666666666" 19:20:49 DEBUG ] 19:20:49 DEBUG "pod": "ds-cts-2" 19:20:49 DEBUG }, 19:20:49 DEBUG 19:20:49 DEBUG ] 19:20:49 DEBUG 19:20:49 DEBUG }, 19:20:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:49 DEBUG }, 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:49 DEBUG "instance": "10.142.0.43:9100" 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG }, 19:20:49 DEBUG "0" 19:20:49 DEBUG 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "value": [ 19:20:49 DEBUG ] 19:20:49 DEBUG "0" 19:20:49 DEBUG }, 19:20:49 DEBUG } 19:20:49 DEBUG "pod": "ds-idrepo-2" 19:20:49 DEBUG "value": [ 19:20:49 DEBUG } 19:20:49 DEBUG 19:20:49 DEBUG } 19:20:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "value": [ 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG }, 19:20:49 DEBUG }, 19:20:49 DEBUG "instance": "10.142.0.43:9100" 19:20:49 DEBUG }, 19:20:49 DEBUG "value": [ 19:20:49 DEBUG ] 19:20:49 DEBUG "pod": "ds-cts-2" 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG "80668523.46666667" 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG "value": [ 19:20:49 DEBUG ] 19:20:49 DEBUG }, 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG ] 19:20:49 DEBUG 19:20:49 DEBUG } 19:20:49 DEBUG }, 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG "2672.8333333333335" 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "value": [ 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG "0.152" 19:20:49 DEBUG ] 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG "0.06800000000000447" 19:20:49 DEBUG "0" 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG } 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "0" 19:20:49 DEBUG } 19:20:49 DEBUG "pod": "ds-cts-2" 19:20:49 DEBUG "value": [ 19:20:49 DEBUG "0" 19:20:49 DEBUG "0" 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG "2671.3333333333335" 19:20:49 DEBUG } 19:20:49 DEBUG }, 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG "0.0001" 19:20:49 DEBUG ] 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG "0" 19:20:49 DEBUG } 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG }, 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG "2666.4666666666667" 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG "3488273529856" 19:20:49 DEBUG "0.18126666666663596" 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG "value": [ 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG "0" 19:20:49 DEBUG } 19:20:49 DEBUG "value": [ 19:20:49 DEBUG ] 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG "480652" 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG ] 19:20:49 DEBUG 1713637249, 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG "0" 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG "30" 19:20:49 DEBUG ] 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG ] 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:20:49 DEBUG } 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637309" 19:21:49 ERROR [http_cmd]: ERROR 19:21:49 ERROR --- status code --- 19:21:49 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:21:49 ERROR --- http response --- 19:21:49 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:21:49 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:21:49 INFO 19:21:49 INFO [loop_until]: kubectl --namespace=xlou top pods 19:21:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO 19:21:49 INFO [loop_until]: kubectl --namespace=xlou top node 19:21:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:21:49 INFO [loop_until]: OK (rc = 0) 19:21:49 DEBUG --- stdout --- 19:21:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16979m 8610Mi am-767f6b67b5-qcqz2 17204m 8703Mi am-767f6b67b5-vbltz 18675m 8665Mi ds-cts-0 18646m 10184Mi ds-cts-1 16585m 10143Mi ds-cts-2 19395m 10182Mi ds-idrepo-0 7374m 3889Mi ds-idrepo-1 7335m 3907Mi ds-idrepo-2 7452m 3753Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 16m 61Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8738m 668Mi 19:21:49 DEBUG --- stderr --- 19:21:49 DEBUG 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637309" 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [loop_until]: OK (rc = 0) 19:21:49 DEBUG --- stdout --- 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 DEBUG --- status code --- 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 DEBUG --- status code --- 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 DEBUG --- status code --- 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 DEBUG --- status code --- 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6219Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7603m 25% 7631Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16872m 56% 11325Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3344Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18094m 60% 13474Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5194Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17185m 57% 11595Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14496m 48% 2963Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19504m 65% 10894Mi 9% gke-xlou-cdm-ds-0f601751-j488 8209m 27% 7141Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17122m 57% 15270Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8186m 27% 7878Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19387m 64% 12931Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9042m 56% 6448Mi 10% 19:21:49 DEBUG --- status code --- 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 DEBUG --- status code --- 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- status code --- 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 INFO [http_cmd]: http status code OK 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG --- stderr --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:49 DEBUG --- status code --- 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG { 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG { 19:21:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG { 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG { 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG --- http response --- 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG { 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG { 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG { 19:21:49 DEBUG { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "data": { 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG { 19:21:49 DEBUG "data": { 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:49 DEBUG "data": { 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG { 19:21:49 DEBUG { 19:21:49 DEBUG { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:21:49 DEBUG "data": { 19:21:49 DEBUG { 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG { 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:21:49 DEBUG { 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG { 19:21:49 DEBUG { 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG { 19:21:49 DEBUG "data": { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "data": { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "data": { 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "data": { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "data": { 19:21:49 DEBUG "data": { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "result": [ 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG { 19:21:49 DEBUG "data": { 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG ----- output ----- 19:21:49 DEBUG "metric": { 19:21:49 DEBUG { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "data": { 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG { 19:21:49 DEBUG "result": [ 19:21:49 DEBUG { 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "data": { 19:21:49 DEBUG "data": { 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "data": { 19:21:49 DEBUG { 19:21:49 DEBUG { 19:21:49 DEBUG "data": { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG { 19:21:49 DEBUG "metric": { 19:21:49 DEBUG { 19:21:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:49 DEBUG "data": { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "data": { 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG { 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "data": { 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "pod": "ds-cts-0" 19:21:49 DEBUG "data": { 19:21:49 DEBUG "status": "success", 19:21:49 DEBUG "data": { 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "data": { 19:21:49 DEBUG { 19:21:49 DEBUG { 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "result": [ 19:21:49 DEBUG { 19:21:49 DEBUG "data": { 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "data": { 19:21:49 DEBUG { 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:21:49 DEBUG "data": { 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG { 19:21:49 DEBUG }, 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG }, 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "result": [ 19:21:49 DEBUG { 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG "metric": { 19:21:49 DEBUG { 19:21:49 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:21:49 DEBUG { 19:21:49 DEBUG { 19:21:49 DEBUG { 19:21:49 DEBUG }, 19:21:49 DEBUG "value": [ 19:21:49 DEBUG { 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "value": [ 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "value": [ 19:21:49 DEBUG "result": [ 19:21:49 DEBUG { 19:21:49 DEBUG { 19:21:49 DEBUG "instance": "10.142.0.107:9100" 19:21:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:49 DEBUG { 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "instance": "10.142.0.107:9100" 19:21:49 DEBUG "result": [ 19:21:49 DEBUG }, 19:21:49 DEBUG "value": [ 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:21:49 DEBUG 19:21:49 DEBUG "pod": "ds-idrepo-0" 19:21:49 DEBUG "result": [ 19:21:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:49 DEBUG 19:21:49 DEBUG { 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG "result": [ 19:21:49 DEBUG { 19:21:49 DEBUG "metric": { 19:21:49 DEBUG }, 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "instance": "10.142.0.107:9100" 19:21:49 DEBUG }, 19:21:49 DEBUG { 19:21:49 DEBUG "result": [ 19:21:49 DEBUG 19:21:49 DEBUG }, 19:21:49 DEBUG "pod": "ds-cts-0" 19:21:49 DEBUG }, 19:21:49 DEBUG "pod": "ds-idrepo-0" 19:21:49 DEBUG "pod": "ds-idrepo-0" 19:21:49 DEBUG }, 19:21:49 DEBUG "resultType": "vector", 19:21:49 DEBUG 19:21:49 DEBUG }, 19:21:49 DEBUG { 19:21:49 DEBUG }, 19:21:49 DEBUG 19:21:49 DEBUG "metric": { 19:21:49 DEBUG 19:21:49 DEBUG [...] 19:21:49 DEBUG }, 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:49 DEBUG "value": [ 19:21:49 DEBUG 19:21:49 DEBUG }, 19:21:49 DEBUG "value": [ 19:21:49 DEBUG "metric": { 19:21:49 DEBUG { 19:21:49 DEBUG [...] 19:21:49 DEBUG "value": [ 19:21:49 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:21:49 DEBUG "value": [ 19:21:49 DEBUG }, 19:21:49 DEBUG }, 19:21:49 DEBUG "value": [ 19:21:49 DEBUG "result": [ 19:21:49 DEBUG 19:21:49 DEBUG "value": [ 19:21:49 DEBUG "metric": { 19:21:49 DEBUG "value": [ 19:21:49 DEBUG 19:21:49 DEBUG "instance": "10.142.0.107:9100" 19:21:49 DEBUG "metric": { 19:21:49 DEBUG 19:21:49 DEBUG "value": [ 19:21:49 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:21:49 DEBUG }, 19:21:49 DEBUG 19:21:49 DEBUG "pod": "ds-idrepo-0" 19:21:49 DEBUG "value": [ 19:21:49 DEBUG 19:21:49 DEBUG "pod": "ds-cts-0" 19:21:49 DEBUG "metric": { 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG }, 19:21:49 DEBUG 19:21:49 DEBUG "value": [ 19:21:49 DEBUG "value": [ 19:21:49 DEBUG 19:21:49 DEBUG { 19:21:49 DEBUG [...] 19:21:49 DEBUG 19:21:49 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:21:49 DEBUG 19:21:49 DEBUG [...] 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG }, 19:21:49 DEBUG "value": [ 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG }, 19:21:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG "value": [ 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG "metric": { 19:21:49 DEBUG 19:21:49 DEBUG }, 19:21:49 DEBUG "value": [ 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG "pod": "am-767f6b67b5-9zw22" 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG "value": [ 19:21:49 DEBUG }, 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG }, 19:21:50 DEBUG "value": [ 19:21:49 DEBUG 19:21:49 DEBUG "value": [ 19:21:49 DEBUG }, 19:21:50 DEBUG "value": [ 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG "instance": "10.142.0.107:9100" 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG "pod": "ds-cts-2" 19:21:49 DEBUG }, 19:21:49 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:49 DEBUG 19:21:49 DEBUG 19:21:49 DEBUG "value": [ 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:49 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG 19:21:49 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG }, 19:21:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG "value": [ 19:21:50 DEBUG }, 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG }, 19:21:50 DEBUG 19:21:50 DEBUG }, 19:21:50 DEBUG 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG }, 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG [...] 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG 19:21:50 DEBUG "2658.6" 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG "instance": "10.142.0.43:9100" 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:21:50 DEBUG 19:21:50 DEBUG "pod": "ds-cts-2" 19:21:50 DEBUG 19:21:50 DEBUG "pod": "ds-idrepo-2" 19:21:50 DEBUG }, 19:21:50 DEBUG [...] 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG "104958.69602246056" 19:21:50 DEBUG }, 19:21:50 DEBUG "value": [ 19:21:50 DEBUG [...] 19:21:50 DEBUG ] 19:21:50 DEBUG "instance": "10.142.0.43:9100" 19:21:50 DEBUG 19:21:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:50 DEBUG 19:21:50 DEBUG }, 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG }, 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "2669.2666666666664" 19:21:50 DEBUG ] 19:21:50 DEBUG }, 19:21:50 DEBUG }, 19:21:50 DEBUG "value": [ 19:21:50 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG ] 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG 19:21:50 DEBUG } 19:21:50 DEBUG }, 19:21:50 DEBUG "value": [ 19:21:50 DEBUG }, 19:21:50 DEBUG 19:21:50 DEBUG "0" 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 19:21:50 DEBUG "instance": "10.142.0.43:9100" 19:21:50 DEBUG 19:21:50 DEBUG "80981548.5" 19:21:50 DEBUG 19:21:50 DEBUG } 19:21:50 DEBUG 19:21:50 DEBUG "value": [ 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG }, 19:21:50 DEBUG "pod": "ds-cts-2" 19:21:50 DEBUG 19:21:50 DEBUG 19:21:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:21:50 DEBUG } 19:21:50 DEBUG "0.152" 19:21:50 DEBUG 19:21:50 DEBUG ] 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "instance": "10.142.0.43:9100" 19:21:50 DEBUG ] 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "pod": "ds-idrepo-2" 19:21:50 DEBUG }, 19:21:50 DEBUG 19:21:50 DEBUG ] 19:21:50 DEBUG 19:21:50 DEBUG ] 19:21:50 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "0" 19:21:50 DEBUG "value": [ 19:21:50 DEBUG }, 19:21:50 DEBUG 19:21:50 DEBUG "pod": "ds-idrepo-2" 19:21:50 DEBUG }, 19:21:50 DEBUG ] 19:21:50 DEBUG ] 19:21:50 DEBUG 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG "2658.233333333333" 19:21:50 DEBUG }, 19:21:50 DEBUG "value": [ 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG }, 19:21:50 DEBUG "value": [ 19:21:50 DEBUG "pod": "ds-cts-2" 19:21:50 DEBUG } 19:21:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:50 DEBUG } 19:21:50 DEBUG }, 19:21:50 DEBUG "17.614554935720257" 19:21:50 DEBUG "0.0006000000000000001" 19:21:50 DEBUG ] 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "802811904" 19:21:50 DEBUG "instance": "10.142.0.43:9100" 19:21:50 DEBUG }, 19:21:50 DEBUG "value": [ 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG 19:21:50 DEBUG } 19:21:50 DEBUG "0" 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG }, 19:21:50 DEBUG ] 19:21:50 DEBUG }, 19:21:50 DEBUG } 19:21:50 DEBUG "value": [ 19:21:50 DEBUG ] 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG }, 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG "0.09033333333332695" 19:21:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:21:50 DEBUG "3491996503040" 19:21:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "80986569.6" 19:21:50 DEBUG "value": [ 19:21:50 DEBUG } 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "8.844622473920207" 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG ] 19:21:50 DEBUG "value": [ 19:21:50 DEBUG "value": [ 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "0" 19:21:50 DEBUG ] 19:21:50 DEBUG ] 19:21:50 DEBUG }, 19:21:50 DEBUG ] 19:21:50 DEBUG }, 19:21:50 DEBUG } 19:21:50 DEBUG "0" 19:21:50 DEBUG ] 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG } 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "2658.8" 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG "0" 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG "value": [ 19:21:50 DEBUG } 19:21:50 DEBUG "value": [ 19:21:50 DEBUG ] 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG "30.666666666666668" 19:21:50 DEBUG ] 19:21:50 DEBUG ] 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG "0" 19:21:50 DEBUG "0.2595333333333959" 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG ] 19:21:50 DEBUG 1713637309, 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG "480366.5" 19:21:50 DEBUG } 19:21:50 DEBUG "0" 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG ] 19:21:50 DEBUG ] 19:21:50 DEBUG ] 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:21:50 DEBUG } 19:22:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637369" 19:22:49 ERROR [http_cmd]: ERROR 19:22:49 ERROR --- status code --- 19:22:49 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:22:49 ERROR --- http response --- 19:22:49 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:22:49 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:22:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637369" 19:22:49 INFO 19:22:49 INFO [loop_until]: kubectl --namespace=xlou top pods 19:22:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:22:49 INFO [http_cmd]: http status code OK 19:22:49 DEBUG --- status code --- 19:22:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:49 DEBUG --- http response --- 19:22:49 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:22:49 INFO [loop_until]: OK (rc = 0) 19:22:49 DEBUG --- stdout --- 19:22:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17049m 8607Mi am-767f6b67b5-qcqz2 17097m 8702Mi am-767f6b67b5-vbltz 18439m 8667Mi ds-cts-0 18900m 10170Mi ds-cts-1 17907m 10131Mi ds-cts-2 18518m 10191Mi ds-idrepo-0 7243m 3889Mi ds-idrepo-1 7507m 3907Mi ds-idrepo-2 7580m 3754Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8694m 672Mi 19:22:49 DEBUG --- stderr --- 19:22:49 DEBUG 19:22:50 INFO 19:22:50 INFO [loop_until]: kubectl --namespace=xlou top node 19:22:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637370" 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637370" 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG --- status code --- 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637370" 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG --- status code --- 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG --- status code --- 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG --- status code --- 19:22:50 INFO [loop_until]: OK (rc = 0) 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG --- status code --- 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG --- status code --- 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637370" 19:22:50 DEBUG --- status code --- 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG --- status code --- 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG --- stdout --- 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6221Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7655m 25% 7624Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16885m 56% 11327Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3350Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19055m 63% 13476Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5191Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17207m 57% 11593Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14509m 48% 2973Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19452m 65% 10890Mi 9% gke-xlou-cdm-ds-0f601751-j488 8200m 27% 7142Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17355m 58% 15277Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8152m 27% 7881Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18790m 62% 12932Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9071m 57% 6449Mi 10% 19:22:50 DEBUG { 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG --- http response --- 19:22:50 INFO [http_cmd]: http status code OK 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG --- stderr --- 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:22:50 DEBUG --- status code --- 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:50 DEBUG { 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG { 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:50 DEBUG { 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG { 19:22:50 DEBUG { 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:50 DEBUG { 19:22:50 DEBUG 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG "data": { 19:22:50 DEBUG { 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG { 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:22:50 DEBUG { 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG { 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:50 DEBUG "data": { 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG { 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG --- http response --- 19:22:50 DEBUG { 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG { 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "data": { 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "data": { 19:22:50 DEBUG "data": { 19:22:50 DEBUG { 19:22:50 DEBUG { 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "data": { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "result": [ 19:22:50 DEBUG { 19:22:50 DEBUG "data": { 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "result": [ 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG { 19:22:50 DEBUG { 19:22:50 DEBUG "data": { 19:22:50 DEBUG "data": { 19:22:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "data": { 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG { 19:22:50 DEBUG { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "data": { 19:22:50 DEBUG { 19:22:50 DEBUG "data": { 19:22:50 DEBUG { 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "data": { 19:22:50 DEBUG { 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "data": { 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "result": [ 19:22:50 DEBUG ----- output ----- 19:22:50 DEBUG "data": { 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "data": { 19:22:50 DEBUG "result": [ 19:22:50 DEBUG { 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "data": { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "data": { 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "data": { 19:22:50 DEBUG { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "result": [ 19:22:50 DEBUG { 19:22:50 DEBUG "metric": { 19:22:50 DEBUG { 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "data": { 19:22:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:22:50 DEBUG }, 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:50 DEBUG }, 19:22:50 DEBUG "data": { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG { 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:50 DEBUG }, 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "instance": "10.142.0.107:9100" 19:22:50 DEBUG { 19:22:50 DEBUG "data": { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:22:50 DEBUG }, 19:22:50 DEBUG { 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG { 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "pod": "ds-cts-0" 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG [...] 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:50 DEBUG "status": "success", 19:22:50 DEBUG "data": { 19:22:50 DEBUG "result": [ 19:22:50 DEBUG { 19:22:50 DEBUG "result": [ 19:22:50 DEBUG }, 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "result": [ 19:22:50 DEBUG { 19:22:50 DEBUG { 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "metric": { 19:22:50 DEBUG }, 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "result": [ 19:22:50 DEBUG }, 19:22:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "data": { 19:22:50 DEBUG { 19:22:50 DEBUG 19:22:50 DEBUG "data": { 19:22:50 DEBUG }, 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "metric": { 19:22:50 DEBUG 19:22:50 DEBUG { 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "metric": { 19:22:50 DEBUG 19:22:50 DEBUG "pod": "ds-cts-0" 19:22:50 DEBUG "value": [ 19:22:50 DEBUG { 19:22:50 DEBUG { 19:22:50 DEBUG "value": [ 19:22:50 DEBUG }, 19:22:50 DEBUG { 19:22:50 DEBUG { 19:22:50 DEBUG { 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "metric": { 19:22:50 DEBUG 19:22:50 DEBUG "resultType": "vector", 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "result": [ 19:22:50 DEBUG { 19:22:50 DEBUG "pod": "ds-idrepo-0" 19:22:50 DEBUG [...] 19:22:50 DEBUG "metric": { 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:22:50 DEBUG { 19:22:50 DEBUG "pod": "ds-idrepo-0" 19:22:50 DEBUG 19:22:50 DEBUG }, 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "metric": { 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "result": [ 19:22:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:50 DEBUG [...] 19:22:50 DEBUG "result": [ 19:22:50 DEBUG 19:22:50 DEBUG { 19:22:50 DEBUG "metric": { 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG }, 19:22:50 DEBUG "metric": { 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "2661" 19:22:50 DEBUG "instance": "10.142.0.107:9100" 19:22:50 DEBUG "instance": "10.142.0.107:9100" 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:22:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:22:50 DEBUG { 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "pod": "ds-cts-0" 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG }, 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "instance": "10.142.0.107:9100" 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG }, 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "pod": "ds-idrepo-0" 19:22:50 DEBUG }, 19:22:50 DEBUG }, 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:22:50 DEBUG "metric": { 19:22:50 DEBUG { 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG [...] 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG [...] 19:22:50 DEBUG [...] 19:22:50 DEBUG [...] 19:22:50 DEBUG ] 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "value": [ 19:22:50 DEBUG [...] 19:22:50 DEBUG [...] 19:22:50 DEBUG "value": [ 19:22:50 DEBUG }, 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:22:50 DEBUG 19:22:50 DEBUG }, 19:22:50 DEBUG "instance": "10.142.0.107:9100" 19:22:50 DEBUG [...] 19:22:50 DEBUG "metric": { 19:22:50 DEBUG "pod": "ds-idrepo-0" 19:22:50 DEBUG }, 19:22:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:22:50 DEBUG 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "pod": "ds-cts-2" 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG } 19:22:50 DEBUG [...] 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG [...] 19:22:50 DEBUG 19:22:50 DEBUG ] 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG [...] 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG [...] 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG [...] 19:22:50 DEBUG "8.80760231186559" 19:22:50 DEBUG 19:22:50 DEBUG }, 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG "value": [ 19:22:50 DEBUG [...] 19:22:50 DEBUG 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:50 DEBUG } 19:22:50 DEBUG 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG [...] 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG [...] 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG ] 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "0" 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG 19:22:50 DEBUG [...] 19:22:50 DEBUG }, 19:22:50 DEBUG } 19:22:50 DEBUG 19:22:50 DEBUG "pod": "ds-cts-2" 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:22:50 DEBUG "instance": "10.142.0.43:9100" 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG [...] 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG [...] 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG 19:22:50 DEBUG [...] 19:22:50 DEBUG "480427.5" 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "0.0001" 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG }, 19:22:50 DEBUG }, 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG [...] 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG } 19:22:50 DEBUG 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG ] 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG ] 19:22:50 DEBUG "pod": "ds-idrepo-2" 19:22:50 DEBUG [...] 19:22:50 DEBUG 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG ] 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG "instance": "10.142.0.43:9100" 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG "pod": "ds-cts-2" 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:50 DEBUG ] 19:22:50 DEBUG 19:22:50 DEBUG "2661.0333333333333" 19:22:50 DEBUG } 19:22:50 DEBUG 19:22:50 DEBUG "pod": "ds-idrepo-2" 19:22:50 DEBUG 19:22:50 DEBUG } 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG "2673.0666666666666" 19:22:50 DEBUG } 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG "instance": "10.142.0.43:9100" 19:22:50 DEBUG }, 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG "2660.7666666666664" 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG }, 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG [...] 19:22:50 DEBUG }, 19:22:50 DEBUG 19:22:50 DEBUG ] 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG ] 19:22:50 DEBUG ] 19:22:50 DEBUG "29.933333333333334" 19:22:50 DEBUG }, 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "16308.309525256822" 19:22:50 DEBUG ] 19:22:50 DEBUG ] 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:22:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:22:50 DEBUG }, 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG "0.152" 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG 19:22:50 DEBUG } 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG "instance": "10.142.0.43:9100" 19:22:50 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG ] 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG "0.10933333333333241" 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG }, 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG "pod": "ds-cts-2" 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG "0" 19:22:50 DEBUG "pod": "ds-idrepo-2" 19:22:50 DEBUG }, 19:22:50 DEBUG "value": [ 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG ] 19:22:50 DEBUG "0.27553333333325913" 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG "0" 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "0" 19:22:50 DEBUG ] 19:22:50 DEBUG "instance": "10.142.0.43:9100" 19:22:50 DEBUG }, 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG }, 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG "0" 19:22:50 DEBUG }, 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG }, 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG "80799115.43333334" 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG "3496301274112" 19:22:50 DEBUG "value": [ 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG ] 19:22:50 DEBUG ] 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG "0" 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG }, 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG "value": [ 19:22:50 DEBUG "0" 19:22:50 DEBUG ] 19:22:50 DEBUG "805142528" 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG "80794583.3" 19:22:50 DEBUG "value": [ 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG 1713637370, 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG "0" 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG "8.84587681350895" 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG ] 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:22:50 DEBUG } 19:23:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637429" 19:23:49 ERROR [http_cmd]: ERROR 19:23:49 ERROR --- status code --- 19:23:49 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:23:49 ERROR --- http response --- 19:23:49 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:23:49 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:23:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637429" 19:23:49 INFO [http_cmd]: http status code OK 19:23:49 DEBUG --- status code --- 19:23:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:49 DEBUG --- http response --- 19:23:49 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:23:49 INFO 19:23:49 INFO [loop_until]: kubectl --namespace=xlou top pods 19:23:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:23:50 INFO [loop_until]: OK (rc = 0) 19:23:50 DEBUG --- stdout --- 19:23:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17173m 8613Mi am-767f6b67b5-qcqz2 16757m 8704Mi am-767f6b67b5-vbltz 19045m 8666Mi ds-cts-0 18411m 10194Mi ds-cts-1 17902m 10129Mi ds-cts-2 19454m 10188Mi ds-idrepo-0 7529m 3889Mi ds-idrepo-1 7341m 3907Mi ds-idrepo-2 7582m 3754Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 15m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8782m 674Mi 19:23:50 DEBUG --- stderr --- 19:23:50 DEBUG 19:23:50 INFO 19:23:50 INFO [loop_until]: kubectl --namespace=xlou top node 19:23:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637430" 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637430" 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637430" 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 DEBUG --- status code --- 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637430" 19:23:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637430" 19:23:50 DEBUG --- status code --- 19:23:50 INFO [loop_until]: OK (rc = 0) 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG --- status code --- 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 DEBUG --- status code --- 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 DEBUG --- stdout --- 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 DEBUG --- status code --- 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG --- status code --- 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 DEBUG --- status code --- 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 DEBUG --- status code --- 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 INFO [http_cmd]: http status code OK 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6220Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7766m 26% 7642Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16997m 56% 11326Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 91m 0% 3353Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18482m 61% 13481Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5180Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17332m 58% 11585Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14634m 49% 2967Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19453m 65% 10889Mi 9% gke-xlou-cdm-ds-0f601751-j488 8242m 27% 7149Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17279m 57% 15248Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8209m 27% 7878Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19195m 64% 12936Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9116m 57% 6450Mi 10% 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG --- status code --- 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:50 DEBUG --- stderr --- 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:50 DEBUG { 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:50 DEBUG { 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:50 DEBUG 19:23:50 DEBUG { 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:50 DEBUG { 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG { 19:23:50 DEBUG { 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG --- http response --- 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG { 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG { 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG { 19:23:50 DEBUG { 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG { 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG { 19:23:50 DEBUG { 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG { 19:23:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:23:50 DEBUG { 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "data": { 19:23:50 DEBUG "data": { 19:23:50 DEBUG { 19:23:50 DEBUG "data": { 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG { 19:23:50 DEBUG "data": { 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG { 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG { 19:23:50 DEBUG "data": { 19:23:50 DEBUG { 19:23:50 DEBUG { 19:23:50 DEBUG { 19:23:50 DEBUG "data": { 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "data": { 19:23:50 DEBUG "data": { 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG ----- output ----- 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG { 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "data": { 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "result": [ 19:23:50 DEBUG { 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "data": { 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "data": { 19:23:50 DEBUG "data": { 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "data": { 19:23:50 DEBUG { 19:23:50 DEBUG "data": { 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "data": { 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG { 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "data": { 19:23:50 DEBUG "data": { 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "data": { 19:23:50 DEBUG "data": { 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "data": { 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "data": { 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "status": "success", 19:23:50 DEBUG "result": [ 19:23:50 DEBUG { 19:23:50 DEBUG { 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "result": [ 19:23:50 DEBUG { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "data": { 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG { 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "data": { 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG { 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "result": [ 19:23:50 DEBUG { 19:23:50 DEBUG { 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "data": { 19:23:50 DEBUG "data": { 19:23:50 DEBUG { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:50 DEBUG { 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "result": [ 19:23:50 DEBUG { 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "metric": { 19:23:50 DEBUG { 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "metric": { 19:23:50 DEBUG { 19:23:50 DEBUG { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG { 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "resultType": "vector", 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:23:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:50 DEBUG { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG }, 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "result": [ 19:23:50 DEBUG { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG { 19:23:50 DEBUG { 19:23:50 DEBUG { 19:23:50 DEBUG "instance": "10.142.0.107:9100" 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "result": [ 19:23:50 DEBUG { 19:23:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:50 DEBUG }, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "result": [ 19:23:50 DEBUG "result": [ 19:23:50 DEBUG { 19:23:50 DEBUG }, 19:23:50 DEBUG }, 19:23:50 DEBUG "pod": "ds-cts-0" 19:23:50 DEBUG "instance": "10.142.0.107:9100" 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:23:50 DEBUG { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG }, 19:23:50 DEBUG "pod": "ds-idrepo-0" 19:23:50 DEBUG { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG }, 19:23:50 DEBUG "instance": "10.142.0.107:9100" 19:23:50 DEBUG "pod": "ds-cts-0" 19:23:50 DEBUG "metric": { 19:23:50 DEBUG 19:23:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:50 DEBUG { 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "pod": "ds-idrepo-0" 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "value": [ 19:23:50 DEBUG }, 19:23:50 DEBUG }, 19:23:50 DEBUG 19:23:50 DEBUG }, 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "pod": "ds-cts-0" 19:23:50 DEBUG }, 19:23:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:23:50 DEBUG "instance": "10.142.0.107:9100" 19:23:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:23:50 DEBUG "value": [ 19:23:50 DEBUG }, 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "pod": "ds-idrepo-0" 19:23:50 DEBUG "value": [ 19:23:50 DEBUG }, 19:23:50 DEBUG }, 19:23:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:23:50 DEBUG 19:23:50 DEBUG }, 19:23:50 DEBUG "metric": { 19:23:50 DEBUG "instance": "10.142.0.107:9100" 19:23:50 DEBUG }, 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "pod": "ds-idrepo-0" 19:23:50 DEBUG }, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG }, 19:23:50 DEBUG }, 19:23:50 DEBUG }, 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:23:50 DEBUG }, 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "value": [ 19:23:50 DEBUG }, 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:23:50 DEBUG }, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG }, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG }, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG }, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG [...] 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG }, 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG }, 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG [...] 19:23:50 DEBUG 19:23:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:23:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:50 DEBUG "pod": "ds-cts-2" 19:23:50 DEBUG 19:23:50 DEBUG }, 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG }, 19:23:50 DEBUG }, 19:23:50 DEBUG "pod": "ds-cts-2" 19:23:50 DEBUG "instance": "10.142.0.43:9100" 19:23:50 DEBUG 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG }, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG "instance": "10.142.0.43:9100" 19:23:50 DEBUG }, 19:23:50 DEBUG [...] 19:23:50 DEBUG }, 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG "pod": "ds-cts-2" 19:23:50 DEBUG 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:23:50 DEBUG 19:23:50 DEBUG 19:23:50 DEBUG "pod": "ds-idrepo-2" 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "value": [ 19:23:50 DEBUG }, 19:23:50 DEBUG }, 19:23:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:23:50 DEBUG "0" 19:23:50 DEBUG 19:23:50 DEBUG "pod": "ds-cts-2" 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:23:50 DEBUG 19:23:50 DEBUG }, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "pod": "ds-idrepo-2" 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG "instance": "10.142.0.43:9100" 19:23:50 DEBUG }, 19:23:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:23:50 DEBUG "2673.233333333333" 19:23:50 DEBUG }, 19:23:50 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:23:50 DEBUG 19:23:50 DEBUG }, 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "value": [ 19:23:50 DEBUG }, 19:23:50 DEBUG ] 19:23:50 DEBUG "pod": "ds-idrepo-2" 19:23:50 DEBUG }, 19:23:50 DEBUG "2671.733333333333" 19:23:50 DEBUG }, 19:23:50 DEBUG 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG }, 19:23:50 DEBUG "2670.3" 19:23:50 DEBUG }, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG }, 19:23:50 DEBUG ] 19:23:50 DEBUG "value": [ 19:23:50 DEBUG }, 19:23:50 DEBUG "instance": "10.142.0.43:9100" 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "8.847459686416043" 19:23:50 DEBUG "2672.866666666667" 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG } 19:23:50 DEBUG }, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG ] 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "instance": "10.142.0.43:9100" 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG "0.20629999999998272" 19:23:50 DEBUG ] 19:23:50 DEBUG "0" 19:23:50 DEBUG "value": [ 19:23:50 DEBUG ] 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG } 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG }, 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG ] 19:23:50 DEBUG ] 19:23:50 DEBUG "0.00009999999999999998" 19:23:50 DEBUG "80857357.06666666" 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG ] 19:23:50 DEBUG "value": [ 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG } 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG }, 19:23:50 DEBUG "0.152" 19:23:50 DEBUG 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG } 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG "24.5" 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG ] 19:23:50 DEBUG "480465.5" 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG "value": [ 19:23:50 DEBUG "0" 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG ] 19:23:50 DEBUG "0" 19:23:50 DEBUG } 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG "0" 19:23:50 DEBUG ] 19:23:50 DEBUG "35656.295511717195" 19:23:50 DEBUG "value": [ 19:23:50 DEBUG ] 19:23:50 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG "0" 19:23:50 DEBUG ] 19:23:50 DEBUG "0.08166666666666816" 19:23:50 DEBUG ] 19:23:50 DEBUG "0" 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG "8.911061375255777" 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG ] 19:23:50 DEBUG ] 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG "0" 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG } 19:23:50 DEBUG }, 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG "3500787784704" 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG "80861967.06666666" 19:23:50 DEBUG ] 19:23:50 DEBUG "value": [ 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG ] 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG 1713637430, 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG "805908480" 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG } 19:23:50 DEBUG ] 19:23:50 DEBUG } 19:23:50 DEBUG } 19:24:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637489" 19:24:49 ERROR [http_cmd]: ERROR 19:24:49 ERROR --- status code --- 19:24:49 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:24:49 ERROR --- http response --- 19:24:49 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:24:49 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:24:50 INFO 19:24:50 INFO [loop_until]: kubectl --namespace=xlou top pods 19:24:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:24:50 INFO [loop_until]: OK (rc = 0) 19:24:50 DEBUG --- stdout --- 19:24:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17206m 8649Mi am-767f6b67b5-qcqz2 16996m 8704Mi am-767f6b67b5-vbltz 18321m 8667Mi ds-cts-0 18859m 10198Mi ds-cts-1 17583m 10132Mi ds-cts-2 19002m 10195Mi ds-idrepo-0 7353m 3889Mi ds-idrepo-1 7383m 3908Mi ds-idrepo-2 7410m 3753Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9013m 711Mi 19:24:50 DEBUG --- stderr --- 19:24:50 DEBUG 19:24:50 INFO 19:24:50 INFO [loop_until]: kubectl --namespace=xlou top node 19:24:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 DEBUG --- status code --- 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637490" 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637490" 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637490" 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 DEBUG --- status code --- 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 DEBUG --- status code --- 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG --- status code --- 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- status code --- 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 INFO [http_cmd]: http status code OK 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:50 DEBUG { 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG --- status code --- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:50 DEBUG { 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG { 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG { 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG { 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:50 DEBUG { 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG { 19:24:50 DEBUG "data": { 19:24:50 DEBUG { 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG { 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "data": { 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG { 19:24:50 DEBUG { 19:24:50 DEBUG { 19:24:50 DEBUG { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG { 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG { 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG --- http response --- 19:24:50 DEBUG { 19:24:50 DEBUG "data": { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG { 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "data": { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "data": { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG { 19:24:50 DEBUG "data": { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:24:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "data": { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "data": { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "data": { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "data": { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "data": { 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "data": { 19:24:50 DEBUG "data": { 19:24:50 DEBUG "data": { 19:24:50 DEBUG "data": { 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "data": { 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "data": { 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG ----- output ----- 19:24:50 DEBUG "data": { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG { 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "data": { 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "data": { 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "data": { 19:24:50 DEBUG { 19:24:50 DEBUG "data": { 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "data": { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG { 19:24:50 DEBUG { 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "metric": { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "metric": { 19:24:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:50 DEBUG "result": [ 19:24:50 DEBUG { 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "result": [ 19:24:50 DEBUG { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "status": "success", 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "metric": { 19:24:50 DEBUG { 19:24:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:50 DEBUG { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG }, 19:24:50 DEBUG { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "metric": { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG { 19:24:50 DEBUG { 19:24:50 DEBUG { 19:24:50 DEBUG "metric": { 19:24:50 DEBUG { 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "metric": { 19:24:50 DEBUG { 19:24:50 DEBUG "data": { 19:24:50 DEBUG "data": { 19:24:50 DEBUG { 19:24:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:50 DEBUG "metric": { 19:24:50 DEBUG }, 19:24:50 DEBUG "metric": { 19:24:50 DEBUG "instance": "10.142.0.107:9100" 19:24:50 DEBUG "metric": { 19:24:50 DEBUG { 19:24:50 DEBUG "metric": { 19:24:50 DEBUG { 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "metric": { 19:24:50 DEBUG { 19:24:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:50 DEBUG { 19:24:50 DEBUG "metric": { 19:24:50 DEBUG "instance": "10.142.0.107:9100" 19:24:50 DEBUG "metric": { 19:24:50 DEBUG "instance": "10.142.0.107:9100" 19:24:50 DEBUG "metric": { 19:24:50 DEBUG { 19:24:50 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:24:50 DEBUG "metric": { 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "resultType": "vector", 19:24:50 DEBUG "metric": { 19:24:50 DEBUG }, 19:24:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG { 19:24:50 DEBUG "instance": "10.142.0.107:9100" 19:24:50 DEBUG "metric": { 19:24:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:50 DEBUG "metric": { 19:24:50 DEBUG 19:24:50 DEBUG "pod": "ds-idrepo-0" 19:24:50 DEBUG "metric": { 19:24:50 DEBUG }, 19:24:50 DEBUG "metric": { 19:24:50 DEBUG }, 19:24:50 DEBUG "metric": { 19:24:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:24:50 DEBUG }, 19:24:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:24:50 DEBUG "metric": { 19:24:50 DEBUG }, 19:24:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "result": [ 19:24:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "metric": { 19:24:50 DEBUG }, 19:24:50 DEBUG "pod": "ds-idrepo-0" 19:24:50 DEBUG }, 19:24:50 DEBUG "pod": "ds-cts-0" 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "pod": "ds-cts-0" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "pod": "ds-idrepo-0" 19:24:50 DEBUG }, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG "instance": "10.142.0.107:9100" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG { 19:24:50 DEBUG { 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG "pod": "ds-idrepo-0" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "metric": { 19:24:50 DEBUG "metric": { 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:24:50 DEBUG "pod": "ds-cts-0" 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG "instance": "10.142.0.43:9100" 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:50 INFO [loop_until]: OK (rc = 0) 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:50 DEBUG 19:24:50 DEBUG "instance": "10.142.0.43:9100" 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG "instance": "10.142.0.43:9100" 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:24:50 DEBUG 19:24:50 DEBUG [...] 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG "instance": "10.142.0.43:9100" 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG --- stdout --- 19:24:50 DEBUG 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "pod": "ds-idrepo-2" 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:24:50 DEBUG }, 19:24:50 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG "pod": "ds-cts-2" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 59m 0% 6227Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7604m 25% 7643Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16954m 56% 11373Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18860m 63% 13483Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5193Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17053m 57% 11598Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14531m 48% 2970Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19505m 65% 10890Mi 9% gke-xlou-cdm-ds-0f601751-j488 8207m 27% 7143Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16894m 56% 15263Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8138m 27% 7869Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19115m 64% 12923Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9154m 57% 6485Mi 11% 19:24:50 DEBUG "2660.5666666666666" 19:24:50 DEBUG }, 19:24:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "pod": "ds-cts-2" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "pod": "ds-idrepo-2" 19:24:50 DEBUG }, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG "instance": "10.142.0.43:9100" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:24:50 DEBUG 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "0" 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "pod": "ds-idrepo-2" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "pod": "ds-cts-2" 19:24:50 DEBUG --- stderr --- 19:24:50 DEBUG ] 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG }, 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG }, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 19:24:50 DEBUG }, 19:24:50 DEBUG "pod": "ds-cts-2" 19:24:50 DEBUG "2659.366666666667" 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG ] 19:24:50 DEBUG "75007126.19999999" 19:24:50 DEBUG }, 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "480509" 19:24:50 DEBUG }, 19:24:50 DEBUG 19:24:50 DEBUG } 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "0.152" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "0.08333333333333144" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "0.19870000000011223" 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "8.916708757139155" 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG }, 19:24:50 DEBUG ] 19:24:50 DEBUG "2659.366666666667" 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "3504737707008" 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "0" 19:24:50 DEBUG "value": [ 19:24:50 DEBUG ] 19:24:50 DEBUG "0" 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG ] 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG ] 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "1110.511831517519" 19:24:50 DEBUG ] 19:24:50 DEBUG "0" 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG ] 19:24:50 DEBUG "2660.2666666666664" 19:24:50 DEBUG }, 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG "value": [ 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG ] 19:24:50 DEBUG ] 19:24:50 DEBUG ] 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG "0" 19:24:50 DEBUG } 19:24:50 DEBUG "30" 19:24:50 DEBUG } 19:24:50 DEBUG "0" 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG "75012015.8" 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG "value": [ 19:24:50 DEBUG "0" 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG "0" 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG "0.00043333333333333326" 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG 1713637490, 19:24:50 DEBUG ] 19:24:50 DEBUG "0" 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG "841994240" 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG ] 19:24:50 DEBUG ] 19:24:50 DEBUG ] 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:24:50 DEBUG } 19:25:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637549" 19:25:50 ERROR [http_cmd]: ERROR 19:25:50 ERROR --- status code --- 19:25:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:25:50 ERROR --- http response --- 19:25:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:25:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:25:50 INFO 19:25:50 INFO [loop_until]: kubectl --namespace=xlou top pods 19:25:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:25:50 INFO [loop_until]: OK (rc = 0) 19:25:50 DEBUG --- stdout --- 19:25:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 7772m 8647Mi am-767f6b67b5-qcqz2 10752m 8520Mi am-767f6b67b5-vbltz 12054m 8664Mi ds-cts-0 14076m 10201Mi ds-cts-1 12515m 10111Mi ds-cts-2 14786m 10261Mi ds-idrepo-0 5270m 3889Mi ds-idrepo-1 3510m 3907Mi ds-idrepo-2 5118m 3753Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 6383m 634Mi 19:25:50 DEBUG --- stderr --- 19:25:50 DEBUG 19:25:50 INFO 19:25:50 INFO [loop_until]: kubectl --namespace=xlou top node 19:25:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637550" 19:25:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637550" 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 DEBUG --- status code --- 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 DEBUG --- status code --- 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 DEBUG --- status code --- 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG --- status code --- 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG --- status code --- 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 DEBUG --- status code --- 19:25:50 INFO [loop_until]: OK (rc = 0) 19:25:50 DEBUG --- status code --- 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 INFO [http_cmd]: http status code OK 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- status code --- 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG --- stdout --- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG { 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG { 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG --- http response --- 19:25:50 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6223Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4040m 13% 7636Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9876m 33% 11367Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3342Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15385m 51% 13476Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5206Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 12038m 40% 11408Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 9392m 31% 2959Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15051m 50% 10879Mi 9% gke-xlou-cdm-ds-0f601751-j488 5737m 19% 7148Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 14528m 48% 15265Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 4843m 16% 7865Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 13921m 46% 12985Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 5573m 35% 6401Mi 10% 19:25:50 DEBUG "data": { 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG { 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG { 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "data": { 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:25:50 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:25:50 DEBUG --- stderr --- 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "data": { 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG "data": { 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG ----- output ----- 19:25:50 DEBUG 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "data": { 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "data": { 19:25:50 DEBUG { 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "data": { 19:25:50 DEBUG "data": { 19:25:50 DEBUG "data": { 19:25:50 DEBUG { 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "result": [ 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "data": { 19:25:50 DEBUG "data": { 19:25:50 DEBUG "data": { 19:25:50 DEBUG "data": { 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "data": { 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "result": [ 19:25:50 DEBUG { 19:25:50 DEBUG "data": { 19:25:50 DEBUG "data": { 19:25:50 DEBUG "data": { 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "result": [ 19:25:50 DEBUG { 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "metric": { 19:25:50 DEBUG { 19:25:50 DEBUG "data": { 19:25:50 DEBUG "data": { 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "data": { 19:25:50 DEBUG { 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "data": { 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "status": "success", 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "data": { 19:25:50 DEBUG { 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "data": { 19:25:50 DEBUG "data": { 19:25:50 DEBUG "pod": "ds-idrepo-0" 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG { 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:50 DEBUG "data": { 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "result": [ 19:25:50 DEBUG { 19:25:50 DEBUG "result": [ 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG }, 19:25:50 DEBUG "pod": "ds-idrepo-0" 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "result": [ 19:25:50 DEBUG { 19:25:50 DEBUG "result": [ 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:50 DEBUG { 19:25:50 DEBUG "result": [ 19:25:50 DEBUG }, 19:25:50 DEBUG "resultType": "vector", 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG "metric": { 19:25:50 DEBUG { 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:25:50 DEBUG }, 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "value": [ 19:25:50 DEBUG }, 19:25:50 DEBUG { 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "metric": { 19:25:50 DEBUG { 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "metric": { 19:25:50 DEBUG { 19:25:50 DEBUG }, 19:25:50 DEBUG "metric": { 19:25:50 DEBUG { 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "result": [ 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "instance": "10.142.0.107:9100" 19:25:50 DEBUG "instance": "10.142.0.107:9100" 19:25:50 DEBUG { 19:25:50 DEBUG }, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG { 19:25:50 DEBUG { 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "instance": "10.142.0.107:9100" 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "pod": "ds-cts-0" 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:25:50 DEBUG "pod": "ds-idrepo-0" 19:25:50 DEBUG }, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "pod": "ds-cts-0" 19:25:50 DEBUG "metric": { 19:25:50 DEBUG 19:25:50 DEBUG { 19:25:50 DEBUG "instance": "10.142.0.107:9100" 19:25:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:50 DEBUG }, 19:25:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:25:50 DEBUG }, 19:25:50 DEBUG }, 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG }, 19:25:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:50 DEBUG "pod": "am-767f6b67b5-9zw22" 19:25:50 DEBUG }, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "pod": "ds-cts-0" 19:25:50 DEBUG 19:25:50 DEBUG "metric": { 19:25:50 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:25:50 DEBUG }, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG }, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "instance": "10.142.0.107:9100" 19:25:50 DEBUG 19:25:50 DEBUG "pod": "ds-idrepo-0" 19:25:50 DEBUG [...] 19:25:50 DEBUG "metric": { 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "value": [ 19:25:50 DEBUG }, 19:25:50 DEBUG }, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:25:50 DEBUG }, 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG }, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG 19:25:50 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG [...] 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG [...] 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG "pod": "ds-cts-2" 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG "pod": "ds-cts-2" 19:25:50 DEBUG [...] 19:25:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG [...] 19:25:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:50 DEBUG [...] 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG }, 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG "pod": "ds-idrepo-2" 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG 19:25:50 DEBUG "value": [ 19:25:50 DEBUG [...] 19:25:50 DEBUG 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:50 DEBUG "instance": "10.142.0.43:9100" 19:25:50 DEBUG "instance": "10.142.0.43:9100" 19:25:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG "pod": "ds-idrepo-2" 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "1701.3666666666666" 19:25:50 DEBUG 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:25:50 DEBUG 19:25:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG 19:25:50 DEBUG "pod": "ds-idrepo-2" 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG }, 19:25:50 DEBUG }, 19:25:50 DEBUG }, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "instance": "10.142.0.43:9100" 19:25:50 DEBUG 19:25:50 DEBUG "value": [ 19:25:50 DEBUG }, 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG ] 19:25:50 DEBUG "instance": "10.142.0.43:9100" 19:25:50 DEBUG "0" 19:25:50 DEBUG 19:25:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:50 DEBUG "1699.1" 19:25:50 DEBUG ] 19:25:50 DEBUG "pod": "ds-cts-2" 19:25:50 DEBUG }, 19:25:50 DEBUG "pod": "ds-cts-2" 19:25:50 DEBUG }, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "value": [ 19:25:50 DEBUG }, 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG }, 19:25:50 DEBUG 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "0" 19:25:50 DEBUG } 19:25:50 DEBUG }, 19:25:50 DEBUG ] 19:25:50 DEBUG 19:25:50 DEBUG }, 19:25:50 DEBUG }, 19:25:50 DEBUG } 19:25:50 DEBUG }, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:25:50 DEBUG "instance": "10.142.0.43:9100" 19:25:50 DEBUG "value": [ 19:25:50 DEBUG }, 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG "78419449.8" 19:25:50 DEBUG ] 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:25:50 DEBUG "0" 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG "0" 19:25:50 DEBUG ] 19:25:50 DEBUG "value": [ 19:25:50 DEBUG } 19:25:50 DEBUG "pod": "am-767f6b67b5-vbltz" 19:25:50 DEBUG }, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG ] 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "25.633333333333333" 19:25:50 DEBUG "0" 19:25:50 DEBUG }, 19:25:50 DEBUG }, 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "2661.0666666666666" 19:25:50 DEBUG "425983" 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG "3507889114112" 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG }, 19:25:50 DEBUG ] 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG "5091165.3025112795" 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG ] 19:25:50 DEBUG ] 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "0" 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG "value": [ 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG "0.10666666666666626" 19:25:50 DEBUG "value": [ 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG "0.2551333333333275" 19:25:50 DEBUG } 19:25:50 DEBUG "value": [ 19:25:50 DEBUG "0" 19:25:50 DEBUG "value": [ 19:25:50 DEBUG ] 19:25:50 DEBUG "0.0004416844444444444" 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG "6.506724402496294" 19:25:50 DEBUG ] 19:25:50 DEBUG "13.144201219256853" 19:25:50 DEBUG ] 19:25:50 DEBUG ] 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG ] 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG ] 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG 1713637550, 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG "0" 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG "761085952" 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG "0.152" 19:25:50 DEBUG } 19:25:50 DEBUG "1699.6333333333332" 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG "78420902.83333334" 19:25:50 DEBUG ] 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG ] 19:25:50 DEBUG ] 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG ] 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:25:50 DEBUG } 19:26:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637610" 19:26:50 ERROR [http_cmd]: ERROR 19:26:50 ERROR --- status code --- 19:26:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:26:50 ERROR --- http response --- 19:26:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:26:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:26:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637610" 19:26:50 INFO [http_cmd]: http status code OK 19:26:50 DEBUG --- status code --- 19:26:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:50 DEBUG --- http response --- 19:26:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:26:50 INFO 19:26:50 INFO [loop_until]: kubectl --namespace=xlou top pods 19:26:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:26:50 INFO [loop_until]: OK (rc = 0) 19:26:50 DEBUG --- stdout --- 19:26:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17131m 8651Mi am-767f6b67b5-qcqz2 16961m 8519Mi am-767f6b67b5-vbltz 18879m 8664Mi ds-cts-0 18378m 10206Mi ds-cts-1 16916m 10135Mi ds-cts-2 19061m 10211Mi ds-idrepo-0 7405m 3890Mi ds-idrepo-1 7439m 3908Mi ds-idrepo-2 7489m 3754Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 17m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8766m 659Mi 19:26:50 DEBUG --- stderr --- 19:26:50 DEBUG 19:26:50 INFO 19:26:50 INFO [loop_until]: kubectl --namespace=xlou top node 19:26:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [loop_until]: OK (rc = 0) 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637611" 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 DEBUG --- stdout --- 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 DEBUG --- status code --- 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 DEBUG --- status code --- 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 DEBUG --- status code --- 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG --- status code --- 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6225Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7663m 25% 7632Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16961m 56% 11362Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3347Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18217m 61% 13483Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5224Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17194m 57% 11415Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14522m 48% 2971Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19444m 65% 10888Mi 9% gke-xlou-cdm-ds-0f601751-j488 8207m 27% 7148Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16558m 55% 15266Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8144m 27% 7875Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18963m 63% 12930Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8807m 55% 6405Mi 10% 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG --- status code --- 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG --- http response --- 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 INFO [http_cmd]: http status code OK 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG --- stderr --- 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG --- status code --- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG { 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "data": { 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG { 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "data": { 19:26:51 DEBUG { 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG { 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG { 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG { 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG { 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:26:51 DEBUG { 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG { 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG "result": [ 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG "data": { 19:26:51 DEBUG { 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG { 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "result": [ 19:26:51 DEBUG { 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG "data": { 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG "data": { 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG "data": { 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG { 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG { 19:26:51 DEBUG { 19:26:51 DEBUG { 19:26:51 DEBUG { 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:26:51 DEBUG "data": { 19:26:51 DEBUG "metric": { 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG { 19:26:51 DEBUG { 19:26:51 DEBUG { 19:26:51 DEBUG "data": { 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG { 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG { 19:26:51 DEBUG "data": { 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "data": { 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG { 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:51 DEBUG }, 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "data": { 19:26:51 DEBUG --- http response --- 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "pod": "ds-idrepo-0" 19:26:51 DEBUG { 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG "result": [ 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG "result": [ 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "result": [ 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "result": [ 19:26:51 DEBUG { 19:26:51 DEBUG "result": [ 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "result": [ 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "data": { 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:26:51 DEBUG "result": [ 19:26:51 DEBUG }, 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "data": { 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "data": { 19:26:51 DEBUG "result": [ 19:26:51 DEBUG { 19:26:51 DEBUG { 19:26:51 DEBUG { 19:26:51 DEBUG { 19:26:51 DEBUG "metric": { 19:26:51 DEBUG { 19:26:51 DEBUG "data": { 19:26:51 DEBUG "data": { 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG { 19:26:51 DEBUG "data": { 19:26:51 DEBUG { 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "data": { 19:26:51 DEBUG "result": [ 19:26:51 DEBUG { 19:26:51 DEBUG { 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "data": { 19:26:51 DEBUG "result": [ 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "data": { 19:26:51 DEBUG { 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "pod": "ds-cts-0" 19:26:51 DEBUG "data": { 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:26:51 DEBUG "data": { 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "metric": { 19:26:51 DEBUG ----- output ----- 19:26:51 DEBUG "metric": { 19:26:51 DEBUG 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "result": [ 19:26:51 DEBUG { 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "data": { 19:26:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "result": [ 19:26:51 DEBUG }, 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "result": [ 19:26:51 DEBUG [...] 19:26:51 DEBUG "result": [ 19:26:51 DEBUG "instance": "10.142.0.107:9100" 19:26:51 DEBUG { 19:26:51 DEBUG "pod": "ds-idrepo-0" 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG { 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "instance": "10.142.0.107:9100" 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "result": [ 19:26:51 DEBUG 19:26:51 DEBUG "data": { 19:26:51 DEBUG }, 19:26:51 DEBUG { 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "pod": "ds-idrepo-0" 19:26:51 DEBUG "result": [ 19:26:51 DEBUG 19:26:51 DEBUG "pod": "ds-idrepo-0" 19:26:51 DEBUG "result": [ 19:26:51 DEBUG "result": [ 19:26:51 DEBUG { 19:26:51 DEBUG 19:26:51 DEBUG { 19:26:51 DEBUG }, 19:26:51 DEBUG "status": "success", 19:26:51 DEBUG "data": { 19:26:51 DEBUG }, 19:26:51 DEBUG 19:26:51 DEBUG "instance": "10.142.0.107:9100" 19:26:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG }, 19:26:51 DEBUG "result": [ 19:26:51 DEBUG { 19:26:51 DEBUG [...] 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG { 19:26:51 DEBUG 19:26:51 DEBUG }, 19:26:51 DEBUG { 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:51 DEBUG 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "result": [ 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "pod": "ds-cts-0" 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG { 19:26:51 DEBUG "metric": { 19:26:51 DEBUG 19:26:51 DEBUG "result": [ 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG "metric": { 19:26:51 DEBUG 19:26:51 DEBUG "metric": { 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG { 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "resultType": "vector", 19:26:51 DEBUG { 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "pod": "ds-idrepo-2" 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG "result": [ 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:26:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:26:51 DEBUG }, 19:26:51 DEBUG { 19:26:51 DEBUG 19:26:51 DEBUG }, 19:26:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:51 DEBUG 19:26:51 DEBUG "pod": "ds-cts-0" 19:26:51 DEBUG 19:26:51 DEBUG "metric": { 19:26:51 DEBUG 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:51 DEBUG "result": [ 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:26:51 DEBUG 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG }, 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "metric": { 19:26:51 DEBUG 19:26:51 DEBUG "value": [ 19:26:51 DEBUG }, 19:26:51 DEBUG [...] 19:26:51 DEBUG }, 19:26:51 DEBUG 19:26:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:26:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG }, 19:26:51 DEBUG 19:26:51 DEBUG { 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG { 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG }, 19:26:51 DEBUG }, 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG "metric": { 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "0" 19:26:51 DEBUG ] 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG "2665.6666666666665" 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG } 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG 19:26:51 DEBUG ] 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG 19:26:51 DEBUG ] 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "instance": "10.142.0.107:9100" 19:26:51 DEBUG "metric": { 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "pod": "ds-cts-2" 19:26:51 DEBUG }, 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:26:51 DEBUG }, 19:26:51 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:26:51 DEBUG 19:26:51 DEBUG "2673.0666666666666" 19:26:51 DEBUG "pod": "ds-idrepo-2" 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG 19:26:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:51 DEBUG "instance": "10.142.0.107:9100" 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG 19:26:51 DEBUG "instance": "10.142.0.43:9100" 19:26:51 DEBUG [...] 19:26:51 DEBUG 19:26:51 DEBUG "value": [ 19:26:51 DEBUG [...] 19:26:51 DEBUG [...] 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "value": [ 19:26:51 DEBUG } 19:26:51 DEBUG }, 19:26:51 DEBUG [...] 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG }, 19:26:51 DEBUG 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "pod": "ds-cts-2" 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG ] 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG }, 19:26:51 DEBUG 19:26:51 DEBUG "pod": "ds-idrepo-2" 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG }, 19:26:51 DEBUG [...] 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:51 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG "0.00003333333333333336" 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "0" 19:26:51 DEBUG ] 19:26:51 DEBUG 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG } 19:26:51 DEBUG "0" 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG "0" 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG "instance": "10.142.0.43:9100" 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG }, 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG "0" 19:26:51 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:26:51 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:26:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG [...] 19:26:51 DEBUG "786571264" 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:51 DEBUG 19:26:51 DEBUG "0" 19:26:51 DEBUG 19:26:51 DEBUG [...] 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG "pod": "ds-cts-2" 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "2665.6666666666665" 19:26:51 DEBUG 19:26:51 DEBUG 19:26:51 DEBUG ] 19:26:51 DEBUG }, 19:26:51 DEBUG }, 19:26:51 DEBUG }, 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG }, 19:26:51 DEBUG "pod": "ds-cts-2" 19:26:51 DEBUG ] 19:26:51 DEBUG }, 19:26:51 DEBUG 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG ] 19:26:51 DEBUG 19:26:51 DEBUG }, 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG ] 19:26:51 DEBUG "instance": "10.142.0.43:9100" 19:26:51 DEBUG 19:26:51 DEBUG } 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "value": [ 19:26:51 DEBUG ] 19:26:51 DEBUG 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG "value": [ 19:26:51 DEBUG }, 19:26:51 DEBUG } 19:26:51 DEBUG "value": [ 19:26:51 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:26:51 DEBUG "0" 19:26:51 DEBUG } 19:26:51 DEBUG [...] 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG } 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG } 19:26:51 DEBUG 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG }, 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG 19:26:51 DEBUG "0.07200000000000462" 19:26:51 DEBUG "0" 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG "0.00003097855591747149" 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG "instance": "10.142.0.43:9100" 19:26:51 DEBUG "411294.5" 19:26:51 DEBUG } 19:26:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG "80803442" 19:26:51 DEBUG "value": [ 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG ] 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG "8.881666014286129" 19:26:51 DEBUG }, 19:26:51 DEBUG ] 19:26:51 DEBUG }, 19:26:51 DEBUG "0.152" 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG } 19:26:51 DEBUG 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG "0.16923333333340432" 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG "value": [ 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG "30" 19:26:51 DEBUG } 19:26:51 DEBUG "0" 19:26:51 DEBUG } 19:26:51 DEBUG 19:26:51 DEBUG "instance": "10.142.0.43:9100" 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG ] 19:26:51 DEBUG ] 19:26:51 DEBUG ] 19:26:51 DEBUG ] 19:26:51 DEBUG }, 19:26:51 DEBUG "value": [ 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG "value": [ 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG "3512209204224" 19:26:51 DEBUG ] 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG 1713637611, 19:26:51 DEBUG } 19:26:51 DEBUG "2667.1" 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG "80808152.1" 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG ] 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:26:51 DEBUG } 19:27:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637670" 19:27:50 ERROR [http_cmd]: ERROR 19:27:50 ERROR --- status code --- 19:27:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:27:50 ERROR --- http response --- 19:27:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:27:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:27:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637670" 19:27:50 INFO [http_cmd]: http status code OK 19:27:50 DEBUG --- status code --- 19:27:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:50 DEBUG --- http response --- 19:27:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:27:50 INFO 19:27:50 INFO [loop_until]: kubectl --namespace=xlou top pods 19:27:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:27:50 INFO [loop_until]: OK (rc = 0) 19:27:50 DEBUG --- stdout --- 19:27:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17571m 8649Mi am-767f6b67b5-qcqz2 17422m 8518Mi am-767f6b67b5-vbltz 18856m 8664Mi ds-cts-0 18728m 10202Mi ds-cts-1 17412m 10140Mi ds-cts-2 18932m 10210Mi ds-idrepo-0 7238m 3890Mi ds-idrepo-1 7422m 3908Mi ds-idrepo-2 7614m 3754Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8799m 667Mi 19:27:50 DEBUG --- stderr --- 19:27:50 DEBUG 19:27:51 INFO 19:27:51 INFO [loop_until]: kubectl --namespace=xlou top node 19:27:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:51 DEBUG { 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG "status": "success", 19:27:51 INFO [loop_until]: OK (rc = 0) 19:27:51 DEBUG { 19:27:51 DEBUG "data": { 19:27:51 DEBUG --- stdout --- 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "data": { 19:27:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6212Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7596m 25% 7643Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16924m 56% 11362Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18448m 61% 13484Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5206Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17359m 58% 11422Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14485m 48% 2956Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19450m 65% 10883Mi 9% gke-xlou-cdm-ds-0f601751-j488 8188m 27% 7158Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16975m 56% 15268Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8135m 27% 7890Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18562m 62% 12939Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9056m 56% 6409Mi 10% 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG --- stderr --- 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637671" 19:27:51 DEBUG { 19:27:51 DEBUG "result": [ 19:27:51 DEBUG 19:27:51 DEBUG "metric": { 19:27:51 DEBUG { 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:51 DEBUG "metric": { 19:27:51 DEBUG }, 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 DEBUG "pod": "ds-idrepo-0" 19:27:51 DEBUG "value": [ 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 DEBUG --- status code --- 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 DEBUG "value": [ 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 DEBUG 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 DEBUG 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637671" 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637671" 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637671" 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637671" 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 DEBUG 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG --- http response --- 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637671" 19:27:51 DEBUG --- status code --- 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637671" 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG [...] 19:27:51 DEBUG --- status code --- 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637671" 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG --- status code --- 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG --- status code --- 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG [...] 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG { 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG { 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG --- http response --- 19:27:51 INFO [http_cmd]: http status code OK 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:27:51 DEBUG 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG }, 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:51 DEBUG "data": { 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:51 DEBUG "data": { 19:27:51 DEBUG --- status code --- 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:51 DEBUG { 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG { 19:27:51 DEBUG "value": [ 19:27:51 DEBUG { 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG { 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:27:51 DEBUG { 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG { 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG { 19:27:51 DEBUG "pod": "ds-idrepo-2" 19:27:51 DEBUG { 19:27:51 DEBUG { 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:27:51 DEBUG { 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG { 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "result": [ 19:27:51 DEBUG { 19:27:51 DEBUG "result": [ 19:27:51 DEBUG --- http response --- 19:27:51 DEBUG { 19:27:51 DEBUG "data": { 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG { 19:27:51 DEBUG { 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG { 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG }, 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "data": { 19:27:51 DEBUG "2669.4" 19:27:51 DEBUG "data": { 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "data": { 19:27:51 DEBUG { 19:27:51 DEBUG "data": { 19:27:51 DEBUG { 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG { 19:27:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "data": { 19:27:51 DEBUG { 19:27:51 DEBUG "data": { 19:27:51 DEBUG { 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "data": { 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "data": { 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "data": { 19:27:51 DEBUG "data": { 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG ] 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG { 19:27:51 DEBUG "data": { 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "metric": { 19:27:51 DEBUG "data": { 19:27:51 DEBUG "metric": { 19:27:51 DEBUG ----- output ----- 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "data": { 19:27:51 DEBUG "data": { 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "result": [ 19:27:51 DEBUG { 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "result": [ 19:27:51 DEBUG } 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "result": [ 19:27:51 DEBUG { 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "pod": "ds-cts-0" 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:51 DEBUG { 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "data": { 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "data": { 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "data": { 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "metric": { 19:27:51 DEBUG "0" 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "result": [ 19:27:51 DEBUG { 19:27:51 DEBUG ] 19:27:51 DEBUG { 19:27:51 DEBUG "data": { 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "data": { 19:27:51 DEBUG "metric": { 19:27:51 DEBUG { 19:27:51 DEBUG }, 19:27:51 DEBUG "result": [ 19:27:51 DEBUG }, 19:27:51 DEBUG "status": "success", 19:27:51 DEBUG { 19:27:51 DEBUG { 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG { 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "result": [ 19:27:51 DEBUG { 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "pod": "ds-idrepo-0" 19:27:51 DEBUG ] 19:27:51 DEBUG { 19:27:51 DEBUG { 19:27:51 DEBUG "metric": { 19:27:51 DEBUG } 19:27:51 DEBUG "metric": { 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG { 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:51 DEBUG "metric": { 19:27:51 DEBUG "value": [ 19:27:51 DEBUG { 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "data": { 19:27:51 DEBUG "metric": { 19:27:51 DEBUG "metric": { 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "metric": { 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "metric": { 19:27:51 DEBUG { 19:27:51 DEBUG "metric": { 19:27:51 DEBUG }, 19:27:51 DEBUG } 19:27:51 DEBUG "metric": { 19:27:51 DEBUG "metric": { 19:27:51 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:27:51 DEBUG } 19:27:51 DEBUG "pod": "ds-cts-0" 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "metric": { 19:27:51 DEBUG "result": [ 19:27:51 DEBUG }, 19:27:51 DEBUG "instance": "10.142.0.107:9100" 19:27:51 DEBUG 19:27:51 DEBUG "metric": { 19:27:51 DEBUG 19:27:51 DEBUG "resultType": "vector", 19:27:51 DEBUG "pod": "ds-idrepo-0" 19:27:51 DEBUG "instance": "10.142.0.107:9100" 19:27:51 DEBUG { 19:27:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:27:51 DEBUG { 19:27:51 DEBUG "result": [ 19:27:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:51 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:27:51 DEBUG "value": [ 19:27:51 DEBUG ] 19:27:51 DEBUG { 19:27:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:27:51 DEBUG }, 19:27:51 DEBUG }, 19:27:51 DEBUG { 19:27:51 DEBUG "pod": "ds-idrepo-0" 19:27:51 DEBUG { 19:27:51 DEBUG "value": [ 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:51 DEBUG 19:27:51 DEBUG "result": [ 19:27:51 DEBUG }, 19:27:51 DEBUG }, 19:27:51 DEBUG "metric": { 19:27:51 DEBUG }, 19:27:51 DEBUG "metric": { 19:27:51 DEBUG { 19:27:51 DEBUG }, 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 DEBUG } 19:27:51 DEBUG "metric": { 19:27:51 DEBUG }, 19:27:51 DEBUG }, 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "metric": { 19:27:51 DEBUG }, 19:27:51 DEBUG "metric": { 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 DEBUG { 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:27:51 DEBUG }, 19:27:51 DEBUG "instance": "10.142.0.107:9100" 19:27:51 DEBUG "metric": { 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG } 19:27:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG "pod": "ds-cts-0" 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "instance": "10.142.0.107:9100" 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG "value": [ 19:27:51 DEBUG [...] 19:27:51 DEBUG "metric": { 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG }, 19:27:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 DEBUG }, 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "instance": "10.142.0.107:9100" 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG [...] 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG "pod": "ds-cts-2" 19:27:51 DEBUG [...] 19:27:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:51 DEBUG 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "pod": "ds-idrepo-2" 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG 19:27:51 DEBUG [...] 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:27:51 DEBUG }, 19:27:51 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "pod": "ds-cts-2" 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG [...] 19:27:51 DEBUG "pod": "ds-idrepo-2" 19:27:51 DEBUG "instance": "10.142.0.43:9100" 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG }, 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG }, 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG "instance": "10.142.0.43:9100" 19:27:51 DEBUG "0.0004" 19:27:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:51 DEBUG "2662.4" 19:27:51 DEBUG 19:27:51 DEBUG }, 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:27:51 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:27:51 DEBUG }, 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG 19:27:51 DEBUG }, 19:27:51 DEBUG "pod": "ds-cts-2" 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "2662.366666666667" 19:27:51 DEBUG }, 19:27:51 DEBUG ] 19:27:51 DEBUG }, 19:27:51 DEBUG ] 19:27:51 DEBUG 19:27:51 DEBUG }, 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG }, 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG }, 19:27:51 DEBUG }, 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "0" 19:27:51 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:27:51 DEBUG "value": [ 19:27:51 DEBUG }, 19:27:51 DEBUG "8.804797052346217" 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG "pod": "ds-cts-2" 19:27:51 DEBUG 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG ] 19:27:51 DEBUG "value": [ 19:27:51 DEBUG } 19:27:51 DEBUG "value": [ 19:27:51 DEBUG } 19:27:51 DEBUG 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "0" 19:27:51 DEBUG "instance": "10.142.0.43:9100" 19:27:51 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:27:51 DEBUG "2660.133333333333" 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG ] 19:27:51 DEBUG }, 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG "value": [ 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG "794599424" 19:27:51 DEBUG }, 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "0" 19:27:51 DEBUG } 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG ] 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG ] 19:27:51 DEBUG "instance": "10.142.0.43:9100" 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG ] 19:27:51 DEBUG }, 19:27:51 DEBUG }, 19:27:51 DEBUG ] 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG "0.152" 19:27:51 DEBUG } 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "0" 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG "0" 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG "instance": "10.142.0.43:9100" 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG "0.2631666666666139" 19:27:51 DEBUG } 19:27:51 DEBUG "411118" 19:27:51 DEBUG } 19:27:51 DEBUG }, 19:27:51 DEBUG "0.10266666666665855" 19:27:51 DEBUG } 19:27:51 DEBUG "value": [ 19:27:51 DEBUG "value": [ 19:27:51 DEBUG } 19:27:51 DEBUG "8.851465773667583" 19:27:51 DEBUG "90971.1721817162" 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG "29.966666666666665" 19:27:51 DEBUG }, 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG "value": [ 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG "0" 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG "value": [ 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG "79386403.23333333" 19:27:51 DEBUG "0" 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG 1713637671, 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG "3516239467520" 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG "79380768.80000001" 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG ] 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:27:51 DEBUG } 19:28:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637730" 19:28:50 ERROR [http_cmd]: ERROR 19:28:50 ERROR --- status code --- 19:28:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:28:50 ERROR --- http response --- 19:28:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:28:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:28:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637730" 19:28:50 INFO [http_cmd]: http status code OK 19:28:50 DEBUG --- status code --- 19:28:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:50 DEBUG --- http response --- 19:28:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:28:50 INFO 19:28:50 INFO [loop_until]: kubectl --namespace=xlou top pods 19:28:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:28:50 INFO [loop_until]: OK (rc = 0) 19:28:50 DEBUG --- stdout --- 19:28:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16041m 8650Mi am-767f6b67b5-qcqz2 15414m 9055Mi am-767f6b67b5-vbltz 17207m 8665Mi ds-cts-0 11908m 10232Mi ds-cts-1 14108m 10168Mi ds-cts-2 12249m 10228Mi ds-idrepo-0 7066m 3893Mi ds-idrepo-1 7067m 3907Mi ds-idrepo-2 6946m 3754Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 5m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 16m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8616m 669Mi 19:28:50 DEBUG --- stderr --- 19:28:50 DEBUG 19:28:51 INFO 19:28:51 INFO [loop_until]: kubectl --namespace=xlou top node 19:28:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [loop_until]: OK (rc = 0) 19:28:51 DEBUG --- stdout --- 19:28:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 60m 0% 6218Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7064m 23% 7628Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 15250m 51% 11365Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3348Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 15643m 52% 13495Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5208Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16119m 53% 11946Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14562m 48% 2960Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18735m 62% 10892Mi 9% gke-xlou-cdm-ds-0f601751-j488 7716m 25% 7156Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13926m 46% 15281Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 7830m 26% 7871Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 14028m 46% 12949Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9115m 57% 6411Mi 10% 19:28:51 DEBUG --- stderr --- 19:28:51 DEBUG 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "data": { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "result": [ 19:28:51 DEBUG { 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "pod": "ds-idrepo-0" 19:28:51 DEBUG }, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "pod": "ds-idrepo-2" 19:28:51 DEBUG }, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG "0" 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "data": { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "result": [ 19:28:51 DEBUG { 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:51 DEBUG }, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:51 DEBUG }, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG "1698.7666666666667" 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "data": { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "result": [ 19:28:51 DEBUG { 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "pod": "ds-idrepo-0" 19:28:51 DEBUG }, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "pod": "ds-idrepo-2" 19:28:51 DEBUG }, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG "0" 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637731" 19:28:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637731" 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG --- status code --- 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG --- status code --- 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG --- status code --- 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG --- status code --- 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG --- status code --- 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG --- status code --- 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG --- status code --- 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 INFO [http_cmd]: http status code OK 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG --- status code --- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG { 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG { 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:51 DEBUG { 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG --- http response --- 19:28:51 DEBUG { 19:28:51 DEBUG { 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG { 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG { 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG { 19:28:51 DEBUG { 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG { 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG { 19:28:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG { 19:28:51 DEBUG { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG "data": { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "data": { 19:28:51 DEBUG { 19:28:51 DEBUG { 19:28:51 DEBUG "data": { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG ----- output ----- 19:28:51 DEBUG "data": { 19:28:51 DEBUG "data": { 19:28:51 DEBUG { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "data": { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "data": { 19:28:51 DEBUG "data": { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "data": { 19:28:51 DEBUG "data": { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "data": { 19:28:51 DEBUG { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "data": { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "data": { 19:28:51 DEBUG { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "data": { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "data": { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "data": { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "data": { 19:28:51 DEBUG "data": { 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "data": { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "result": [ 19:28:51 DEBUG { 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "data": { 19:28:51 DEBUG { 19:28:51 DEBUG "result": [ 19:28:51 DEBUG { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG { 19:28:51 DEBUG "status": "success", 19:28:51 DEBUG { 19:28:51 DEBUG { 19:28:51 DEBUG { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "data": { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG { 19:28:51 DEBUG "result": [ 19:28:51 DEBUG { 19:28:51 DEBUG { 19:28:51 DEBUG "result": [ 19:28:51 DEBUG { 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "metric": { 19:28:51 DEBUG { 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "data": { 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:28:51 DEBUG { 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "metric": { 19:28:51 DEBUG { 19:28:51 DEBUG { 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "metric": { 19:28:51 DEBUG { 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "result": [ 19:28:51 DEBUG "pod": "ds-idrepo-0" 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:51 DEBUG { 19:28:51 DEBUG { 19:28:51 DEBUG "pod": "ds-idrepo-0" 19:28:51 DEBUG "resultType": "vector", 19:28:51 DEBUG "instance": "10.142.0.107:9100" 19:28:51 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:28:51 DEBUG "instance": "10.142.0.107:9100" 19:28:51 DEBUG { 19:28:51 DEBUG }, 19:28:51 DEBUG "metric": { 19:28:51 DEBUG { 19:28:51 DEBUG "pod": "ds-cts-0" 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:28:51 DEBUG { 19:28:51 DEBUG }, 19:28:51 DEBUG "pod": "ds-cts-0" 19:28:51 DEBUG }, 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "metric": { 19:28:51 DEBUG }, 19:28:51 DEBUG "result": [ 19:28:51 DEBUG }, 19:28:51 DEBUG }, 19:28:51 DEBUG }, 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:51 DEBUG "metric": { 19:28:51 DEBUG }, 19:28:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:28:51 DEBUG "pod": "ds-cts-0" 19:28:51 DEBUG }, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG "metric": { 19:28:51 DEBUG "value": [ 19:28:51 DEBUG }, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:28:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:51 DEBUG "value": [ 19:28:51 DEBUG { 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "instance": "10.142.0.107:9100" 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG "instance": "10.142.0.107:9100" 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "instance": "10.142.0.107:9100" 19:28:51 DEBUG 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG "metric": { 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG "value": [ 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "value": [ 19:28:51 DEBUG [...] 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG "pod": "ds-cts-2" 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG "pod": "ds-cts-0" 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG [...] 19:28:51 DEBUG "pod": "ds-cts-0" 19:28:51 DEBUG "pod": "ds-idrepo-2" 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG "pod": "ds-cts-2" 19:28:51 DEBUG [...] 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "instance": "10.142.0.43:9100" 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG }, 19:28:51 DEBUG "pod": "ds-cts-2" 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG 19:28:51 DEBUG "instance": "10.142.0.43:9100" 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG "pod": "ds-cts-0" 19:28:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG 19:28:51 DEBUG 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "value": [ 19:28:51 DEBUG }, 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 19:28:51 DEBUG }, 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG }, 19:28:51 DEBUG "instance": "10.142.0.43:9100" 19:28:51 DEBUG 19:28:51 DEBUG "0.0009266511111111111" 19:28:51 DEBUG }, 19:28:51 DEBUG }, 19:28:51 DEBUG "2671.866666666667" 19:28:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:28:51 DEBUG 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "0.152" 19:28:51 DEBUG }, 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG 19:28:51 DEBUG "8.901779142809177" 19:28:51 DEBUG }, 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG "instance": "10.142.0.43:9100" 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "7.849907675848074" 19:28:51 DEBUG "value": [ 19:28:51 DEBUG }, 19:28:51 DEBUG "pod": "ds-cts-2" 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG "value": [ 19:28:51 DEBUG ] 19:28:51 DEBUG }, 19:28:51 DEBUG "instance": "10.142.0.43:9100" 19:28:51 DEBUG "344.5865026964366" 19:28:51 DEBUG "0" 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG ] 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "0" 19:28:51 DEBUG "pod": "ds-cts-0" 19:28:51 DEBUG ] 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "0.1626000000000469" 19:28:51 DEBUG }, 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG "value": [ 19:28:51 DEBUG }, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG "98436.87760361539" 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG }, 19:28:51 DEBUG ] 19:28:51 DEBUG ] 19:28:51 DEBUG "0" 19:28:51 DEBUG } 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG ] 19:28:51 DEBUG }, 19:28:51 DEBUG } 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG ] 19:28:51 DEBUG "value": [ 19:28:51 DEBUG "0.06599999999999683" 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG ] 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG "value": [ 19:28:51 DEBUG } 19:28:51 DEBUG "value": [ 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG ] 19:28:51 DEBUG "465286.5" 19:28:51 DEBUG } 19:28:51 DEBUG "value": [ 19:28:51 DEBUG ] 19:28:51 DEBUG "797708288" 19:28:51 DEBUG } 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG ] 19:28:51 DEBUG "0" 19:28:51 DEBUG } 19:28:51 DEBUG "80259843.9" 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG "2671.6" 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG ] 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG ] 19:28:51 DEBUG 1713637731, 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG ] 19:28:51 DEBUG "80254796" 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG "29.966666666666665" 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG ] 19:28:51 DEBUG "2672.8333333333335" 19:28:51 DEBUG "3519774456832" 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG "29533130.800100796" 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG ] 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:28:51 DEBUG } 19:29:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637790" 19:29:50 ERROR [http_cmd]: ERROR 19:29:50 ERROR --- status code --- 19:29:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:29:50 ERROR --- http response --- 19:29:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:29:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:29:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637790" 19:29:50 INFO [http_cmd]: http status code OK 19:29:50 DEBUG --- status code --- 19:29:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:50 DEBUG --- http response --- 19:29:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:29:50 INFO 19:29:50 INFO [loop_until]: kubectl --namespace=xlou top pods 19:29:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:29:50 INFO [loop_until]: OK (rc = 0) 19:29:50 DEBUG --- stdout --- 19:29:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17108m 8650Mi am-767f6b67b5-qcqz2 17523m 9059Mi am-767f6b67b5-vbltz 18486m 8665Mi ds-cts-0 18886m 10219Mi ds-cts-1 17614m 10149Mi ds-cts-2 19228m 10208Mi ds-idrepo-0 7373m 3892Mi ds-idrepo-1 7324m 3908Mi ds-idrepo-2 7535m 3753Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 13m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9040m 704Mi 19:29:50 DEBUG --- stderr --- 19:29:50 DEBUG 19:29:51 INFO 19:29:51 INFO [loop_until]: kubectl --namespace=xlou top node 19:29:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG { 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "data": { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "result": [ 19:29:51 DEBUG { 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "pod": "ds-idrepo-0" 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "pod": "ds-idrepo-2" 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG "0" 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG { 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "data": { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "result": [ 19:29:51 DEBUG { 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG "2668.7666666666664" 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG { 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "data": { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "result": [ 19:29:51 DEBUG { 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "pod": "ds-idrepo-0" 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "pod": "ds-idrepo-2" 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG "0" 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 INFO [loop_until]: OK (rc = 0) 19:29:51 DEBUG --- stdout --- 19:29:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6208Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7637m 25% 7630Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17002m 56% 11372Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3355Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18751m 62% 13487Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 141m 0% 5234Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17263m 57% 11946Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14598m 48% 2966Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19440m 65% 10895Mi 9% gke-xlou-cdm-ds-0f601751-j488 8248m 27% 7164Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17421m 58% 15261Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8191m 27% 7874Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19339m 64% 12940Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9194m 57% 6444Mi 10% 19:29:51 DEBUG --- stderr --- 19:29:51 DEBUG 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637791" 19:29:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637791" 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 DEBUG --- status code --- 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 DEBUG --- status code --- 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG --- status code --- 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG --- status code --- 19:29:51 INFO [http_cmd]: http status code OK 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- status code --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:29:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:29:51 DEBUG { 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:29:51 DEBUG --- http response --- 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG { 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:29:51 DEBUG { 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG { 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG { 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:29:51 DEBUG { 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG { 19:29:51 DEBUG "data": { 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG { 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG { 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG { 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG { 19:29:51 DEBUG "data": { 19:29:51 DEBUG { 19:29:51 DEBUG { 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG "data": { 19:29:51 DEBUG { 19:29:51 DEBUG { 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG ----- output ----- 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG { 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG { 19:29:51 DEBUG "data": { 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "data": { 19:29:51 DEBUG { 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "data": { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG { 19:29:51 DEBUG { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "data": { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "data": { 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "data": { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "data": { 19:29:51 DEBUG "data": { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "data": { 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "data": { 19:29:51 DEBUG "data": { 19:29:51 DEBUG { 19:29:51 DEBUG { 19:29:51 DEBUG { 19:29:51 DEBUG "data": { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG { 19:29:51 DEBUG "data": { 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "data": { 19:29:51 DEBUG { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "data": { 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "data": { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "status": "success", 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "data": { 19:29:51 DEBUG { 19:29:51 DEBUG { 19:29:51 DEBUG "result": [ 19:29:51 DEBUG { 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "result": [ 19:29:51 DEBUG { 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "data": { 19:29:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:51 DEBUG "data": { 19:29:51 DEBUG { 19:29:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "instance": "10.142.0.107:9100" 19:29:51 DEBUG { 19:29:51 DEBUG "metric": { 19:29:51 DEBUG { 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:51 DEBUG { 19:29:51 DEBUG "result": [ 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "result": [ 19:29:51 DEBUG { 19:29:51 DEBUG "metric": { 19:29:51 DEBUG { 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG }, 19:29:51 DEBUG "resultType": "vector", 19:29:51 DEBUG "metric": { 19:29:51 DEBUG }, 19:29:51 DEBUG { 19:29:51 DEBUG { 19:29:51 DEBUG "result": [ 19:29:51 DEBUG }, 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "pod": "ds-cts-0" 19:29:51 DEBUG "metric": { 19:29:51 DEBUG { 19:29:51 DEBUG }, 19:29:51 DEBUG "metric": { 19:29:51 DEBUG { 19:29:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:51 DEBUG { 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "result": [ 19:29:51 DEBUG { 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:29:51 DEBUG "value": [ 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "metric": { 19:29:51 DEBUG { 19:29:51 DEBUG "value": [ 19:29:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:29:51 DEBUG }, 19:29:51 DEBUG "pod": "ds-idrepo-0" 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "value": [ 19:29:51 DEBUG "instance": "10.142.0.107:9100" 19:29:51 DEBUG "metric": { 19:29:51 DEBUG }, 19:29:51 DEBUG "metric": { 19:29:51 DEBUG "pod": "ds-cts-0" 19:29:51 DEBUG }, 19:29:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:29:51 DEBUG "result": [ 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "pod": "ds-idrepo-0" 19:29:51 DEBUG "instance": "10.142.0.107:9100" 19:29:51 DEBUG "metric": { 19:29:51 DEBUG 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG }, 19:29:51 DEBUG 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG "value": [ 19:29:51 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG }, 19:29:51 DEBUG { 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG "instance": "10.142.0.107:9100" 19:29:51 DEBUG 19:29:51 DEBUG }, 19:29:51 DEBUG }, 19:29:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "pod": "ds-cts-0" 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "value": [ 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG "metric": { 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG }, 19:29:51 DEBUG 19:29:51 DEBUG "value": [ 19:29:51 DEBUG "value": [ 19:29:51 DEBUG }, 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG }, 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "instance": "10.142.0.107:9100" 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG "value": [ 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "value": [ 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG }, 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "value": [ 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:29:51 DEBUG }, 19:29:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG [...] 19:29:51 DEBUG "instance": "10.142.0.43:9100" 19:29:51 DEBUG 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:51 DEBUG 19:29:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:51 DEBUG }, 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:29:51 DEBUG 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG "value": [ 19:29:51 DEBUG [...] 19:29:51 DEBUG }, 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG }, 19:29:51 DEBUG "pod": "ds-cts-2" 19:29:51 DEBUG [...] 19:29:51 DEBUG 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:29:51 DEBUG }, 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG 19:29:51 DEBUG "value": [ 19:29:51 DEBUG "pod": "ds-idrepo-2" 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG "value": [ 19:29:51 DEBUG }, 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG }, 19:29:51 DEBUG 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG 19:29:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:29:51 DEBUG "0" 19:29:51 DEBUG ] 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG "0.152" 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG "pod": "ds-cts-2" 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG "0" 19:29:51 DEBUG "2665.733333333333" 19:29:51 DEBUG "instance": "10.142.0.43:9100" 19:29:51 DEBUG "2669.2666666666664" 19:29:51 DEBUG "value": [ 19:29:51 DEBUG [...] 19:29:51 DEBUG "0" 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG }, 19:29:51 DEBUG } 19:29:51 DEBUG 19:29:51 DEBUG 19:29:51 DEBUG }, 19:29:51 DEBUG "instance": "10.142.0.43:9100" 19:29:51 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:29:51 DEBUG }, 19:29:51 DEBUG "0" 19:29:51 DEBUG 19:29:51 DEBUG }, 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:29:51 DEBUG ] 19:29:51 DEBUG ] 19:29:51 DEBUG }, 19:29:51 DEBUG ] 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG 19:29:51 DEBUG ] 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 19:29:51 DEBUG ] 19:29:51 DEBUG "instance": "10.142.0.43:9100" 19:29:51 DEBUG }, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG ] 19:29:51 DEBUG 19:29:51 DEBUG "value": [ 19:29:51 DEBUG "5.6046628374048835" 19:29:51 DEBUG }, 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG "value": [ 19:29:51 DEBUG } 19:29:51 DEBUG "834134016" 19:29:51 DEBUG 19:29:51 DEBUG } 19:29:51 DEBUG "2666.866666666667" 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG 19:29:51 DEBUG } 19:29:51 DEBUG "value": [ 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG "0" 19:29:51 DEBUG }, 19:29:51 DEBUG "20652329.836353067" 19:29:51 DEBUG } 19:29:51 DEBUG "0.23783333333325157" 19:29:51 DEBUG "pod": "ds-cts-2" 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG ] 19:29:51 DEBUG "value": [ 19:29:51 DEBUG ] 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG ] 19:29:51 DEBUG 19:29:51 DEBUG ] 19:29:51 DEBUG ] 19:29:51 DEBUG "480174.5" 19:29:51 DEBUG "instance": "10.142.0.43:9100" 19:29:51 DEBUG } 19:29:51 DEBUG "80978435.5" 19:29:51 DEBUG ] 19:29:51 DEBUG "value": [ 19:29:51 DEBUG ] 19:29:51 DEBUG ] 19:29:51 DEBUG ] 19:29:51 DEBUG ] 19:29:51 DEBUG }, 19:29:51 DEBUG "0" 19:29:51 DEBUG } 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG "3523972193280" 19:29:51 DEBUG } 19:29:51 DEBUG "pod": "ds-cts-2" 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG }, 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG "value": [ 19:29:51 DEBUG ] 19:29:51 DEBUG ] 19:29:51 DEBUG "8.97393363484034" 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG }, 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG "value": [ 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG "0.08866666666666748" 19:29:51 DEBUG ] 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG "value": [ 19:29:51 DEBUG ] 19:29:51 DEBUG ] 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG "29.933333333333334" 19:29:51 DEBUG ] 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG "80974232.53333333" 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG 1713637791, 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG "0.0009333333333333333" 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG ] 19:29:51 DEBUG } 19:29:51 DEBUG } 19:29:51 DEBUG } 19:30:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637850" 19:30:50 ERROR [http_cmd]: ERROR 19:30:50 ERROR --- status code --- 19:30:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:30:50 ERROR --- http response --- 19:30:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:30:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:30:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637850" 19:30:50 INFO [http_cmd]: http status code OK 19:30:50 DEBUG --- status code --- 19:30:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:50 DEBUG --- http response --- 19:30:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:30:50 INFO 19:30:50 INFO [loop_until]: kubectl --namespace=xlou top pods 19:30:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:30:51 INFO [loop_until]: OK (rc = 0) 19:30:51 DEBUG --- stdout --- 19:30:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17113m 8651Mi am-767f6b67b5-qcqz2 17237m 9061Mi am-767f6b67b5-vbltz 18555m 8665Mi ds-cts-0 18248m 10224Mi ds-cts-1 16844m 10168Mi ds-cts-2 18836m 10212Mi ds-idrepo-0 7428m 3892Mi ds-idrepo-1 7573m 3908Mi ds-idrepo-2 7450m 3753Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 15m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8779m 706Mi 19:30:51 DEBUG --- stderr --- 19:30:51 DEBUG 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: http status code OK 19:30:51 DEBUG --- status code --- 19:30:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:51 DEBUG --- http response --- 19:30:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:51 DEBUG ----- output ----- 19:30:51 DEBUG { 19:30:51 DEBUG "status": "success", 19:30:51 DEBUG "data": { 19:30:51 DEBUG "resultType": "vector", 19:30:51 DEBUG "result": [ 19:30:51 DEBUG { 19:30:51 DEBUG "metric": { 19:30:51 DEBUG "pod": "ds-idrepo-0" 19:30:51 DEBUG }, 19:30:51 DEBUG "value": [ 19:30:51 DEBUG 19:30:51 DEBUG 19:30:51 DEBUG 19:30:51 DEBUG [...] 19:30:51 DEBUG 19:30:51 DEBUG 19:30:51 DEBUG 19:30:51 DEBUG "pod": "ds-idrepo-2" 19:30:51 DEBUG }, 19:30:51 DEBUG "value": [ 19:30:51 DEBUG 1713637851, 19:30:51 DEBUG "0" 19:30:51 DEBUG ] 19:30:51 DEBUG } 19:30:51 DEBUG ] 19:30:51 DEBUG } 19:30:51 DEBUG } 19:30:51 INFO [http_cmd]: http status code OK 19:30:51 DEBUG --- status code --- 19:30:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:51 DEBUG --- http response --- 19:30:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:51 DEBUG ----- output ----- 19:30:51 DEBUG { 19:30:51 DEBUG "status": "success", 19:30:51 DEBUG "data": { 19:30:51 DEBUG "resultType": "vector", 19:30:51 DEBUG "result": [ 19:30:51 INFO [http_cmd]: http status code OK 19:30:51 DEBUG { 19:30:51 DEBUG --- status code --- 19:30:51 DEBUG "metric": { 19:30:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:51 DEBUG --- http response --- 19:30:51 DEBUG }, 19:30:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:51 DEBUG "value": [ 19:30:51 DEBUG ----- output ----- 19:30:51 DEBUG 19:30:51 DEBUG { 19:30:51 DEBUG 19:30:51 DEBUG "status": "success", 19:30:51 DEBUG 19:30:51 DEBUG "data": { 19:30:51 DEBUG [...] 19:30:51 DEBUG "resultType": "vector", 19:30:51 DEBUG 19:30:51 DEBUG "result": [ 19:30:51 DEBUG 19:30:51 DEBUG { 19:30:51 DEBUG 19:30:51 DEBUG "metric": { 19:30:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:51 DEBUG "pod": "ds-idrepo-0" 19:30:51 DEBUG }, 19:30:51 DEBUG }, 19:30:51 DEBUG "value": [ 19:30:51 DEBUG "value": [ 19:30:51 DEBUG 1713637851, 19:30:51 DEBUG 19:30:51 DEBUG "2659.9666666666667" 19:30:51 DEBUG 19:30:51 DEBUG ] 19:30:51 DEBUG 19:30:51 DEBUG } 19:30:51 DEBUG [...] 19:30:51 DEBUG ] 19:30:51 DEBUG 19:30:51 DEBUG } 19:30:51 DEBUG 19:30:51 DEBUG } 19:30:51 DEBUG 19:30:51 DEBUG "pod": "ds-idrepo-2" 19:30:51 DEBUG }, 19:30:51 DEBUG "value": [ 19:30:51 DEBUG 1713637851, 19:30:51 DEBUG "0" 19:30:51 DEBUG ] 19:30:51 DEBUG } 19:30:51 DEBUG ] 19:30:51 DEBUG } 19:30:51 DEBUG } 19:30:51 INFO 19:30:51 INFO [loop_until]: kubectl --namespace=xlou top node 19:30:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:30:51 INFO [loop_until]: OK (rc = 0) 19:30:51 DEBUG --- stdout --- 19:30:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6210Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7641m 25% 7637Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17023m 57% 11376Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3345Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18406m 61% 13487Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5207Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17259m 57% 11950Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14580m 48% 2955Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18897m 63% 10891Mi 9% gke-xlou-cdm-ds-0f601751-j488 8235m 27% 7144Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16734m 56% 15277Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8185m 27% 7871Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18936m 63% 12943Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9110m 57% 6447Mi 10% 19:30:51 DEBUG --- stderr --- 19:30:51 DEBUG 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637851" 19:30:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637851" 19:30:51 INFO [http_cmd]: http status code OK 19:30:51 INFO [http_cmd]: http status code OK 19:30:51 INFO [http_cmd]: http status code OK 19:30:51 INFO [http_cmd]: http status code OK 19:30:51 INFO [http_cmd]: http status code OK 19:30:51 INFO [http_cmd]: http status code OK 19:30:51 INFO [http_cmd]: http status code OK 19:30:52 DEBUG --- status code --- 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 DEBUG --- status code --- 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG --- status code --- 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 DEBUG --- status code --- 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG --- status code --- 19:30:52 INFO [http_cmd]: http status code OK 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG --- status code --- 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG { 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG { 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:52 DEBUG { 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG { 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG --- http response --- 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG { 19:30:52 DEBUG { 19:30:52 DEBUG { 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:52 DEBUG { 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG { 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG { 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG { 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:52 DEBUG { 19:30:52 DEBUG { 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG { 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG "data": { 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG "data": { 19:30:52 DEBUG { 19:30:52 DEBUG "data": { 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:30:52 DEBUG "data": { 19:30:52 DEBUG "data": { 19:30:52 DEBUG "data": { 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG "data": { 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG "data": { 19:30:52 DEBUG { 19:30:52 DEBUG { 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "result": [ 19:30:52 DEBUG { 19:30:52 DEBUG "metric": { 19:30:52 DEBUG "instance": "10.142.0.107:9100" 19:30:52 DEBUG "data": { 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG { 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG { 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG { 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG "data": { 19:30:52 DEBUG "data": { 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "data": { 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG "data": { 19:30:52 DEBUG }, 19:30:52 DEBUG ----- output ----- 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "result": [ 19:30:52 DEBUG { 19:30:52 DEBUG "data": { 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG "data": { 19:30:52 DEBUG "data": { 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "result": [ 19:30:52 DEBUG "data": { 19:30:52 DEBUG "result": [ 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "result": [ 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "result": [ 19:30:52 DEBUG { 19:30:52 DEBUG "result": [ 19:30:52 DEBUG "result": [ 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG { 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG "data": { 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "result": [ 19:30:52 DEBUG { 19:30:52 DEBUG "metric": { 19:30:52 DEBUG "pod": "ds-idrepo-0" 19:30:52 DEBUG "data": { 19:30:52 DEBUG { 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "result": [ 19:30:52 DEBUG { 19:30:52 DEBUG "metric": { 19:30:52 DEBUG "instance": "10.142.0.107:9100" 19:30:52 DEBUG { 19:30:52 DEBUG "metric": { 19:30:52 DEBUG { 19:30:52 DEBUG "metric": { 19:30:52 DEBUG "result": [ 19:30:52 DEBUG 19:30:52 DEBUG "data": { 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "result": [ 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG "metric": { 19:30:52 DEBUG "pod": "ds-cts-0" 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "metric": { 19:30:52 DEBUG "instance": "10.142.0.107:9100" 19:30:52 DEBUG }, 19:30:52 DEBUG { 19:30:52 DEBUG }, 19:30:52 DEBUG { 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "result": [ 19:30:52 DEBUG "result": [ 19:30:52 DEBUG "instance": "10.142.0.107:9100" 19:30:52 DEBUG { 19:30:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:52 DEBUG { 19:30:52 DEBUG 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "result": [ 19:30:52 DEBUG { 19:30:52 DEBUG }, 19:30:52 DEBUG "data": { 19:30:52 DEBUG }, 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "result": [ 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "result": [ 19:30:52 DEBUG "metric": { 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "metric": { 19:30:52 DEBUG "status": "success", 19:30:52 DEBUG { 19:30:52 DEBUG "metric": { 19:30:52 DEBUG }, 19:30:52 DEBUG "metric": { 19:30:52 DEBUG }, 19:30:52 DEBUG "metric": { 19:30:52 DEBUG 19:30:52 DEBUG "result": [ 19:30:52 DEBUG { 19:30:52 DEBUG "metric": { 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG { 19:30:52 DEBUG 19:30:52 DEBUG "result": [ 19:30:52 DEBUG { 19:30:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:52 DEBUG 19:30:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:30:52 DEBUG "data": { 19:30:52 DEBUG "pod": "ds-idrepo-0" 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG [...] 19:30:52 DEBUG 19:30:52 DEBUG "metric": { 19:30:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "metric": { 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "metric": { 19:30:52 DEBUG }, 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG [...] 19:30:52 DEBUG }, 19:30:52 DEBUG { 19:30:52 DEBUG "metric": { 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG { 19:30:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:30:52 DEBUG 19:30:52 DEBUG [...] 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG { 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "resultType": "vector", 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "pod": "ds-cts-0" 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "instance": "10.142.0.43:9100" 19:30:52 DEBUG }, 19:30:52 DEBUG 19:30:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:52 DEBUG "result": [ 19:30:52 DEBUG "result": [ 19:30:52 DEBUG "metric": { 19:30:52 DEBUG [...] 19:30:52 DEBUG 19:30:52 DEBUG [...] 19:30:52 DEBUG }, 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG [...] 19:30:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG [...] 19:30:52 DEBUG { 19:30:52 DEBUG "metric": { 19:30:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:30:52 DEBUG }, 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG [...] 19:30:52 DEBUG "result": [ 19:30:52 DEBUG [...] 19:30:52 DEBUG }, 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG [...] 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG "0.06766666666669229" 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "metric": { 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG { 19:30:52 DEBUG 19:30:52 DEBUG "instance": "10.142.0.43:9100" 19:30:52 DEBUG 19:30:52 DEBUG [...] 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "pod": "ds-cts-0" 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG }, 19:30:52 DEBUG "instance": "10.142.0.107:9100" 19:30:52 DEBUG [...] 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "pod": "ds-cts-2" 19:30:52 DEBUG { 19:30:52 DEBUG "value": [ 19:30:52 DEBUG }, 19:30:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG } 19:30:52 DEBUG }, 19:30:52 DEBUG 19:30:52 DEBUG [...] 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "value": [ 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG "metric": { 19:30:52 DEBUG 19:30:52 DEBUG }, 19:30:52 DEBUG "metric": { 19:30:52 DEBUG 19:30:52 DEBUG "value": [ 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG "instance": "10.142.0.43:9100" 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG } 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "instance": "10.142.0.43:9100" 19:30:52 DEBUG }, 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG "2670.733333333333" 19:30:52 DEBUG ] 19:30:52 DEBUG 19:30:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:30:52 DEBUG 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:52 DEBUG [...] 19:30:52 DEBUG 19:30:52 DEBUG [...] 19:30:52 DEBUG 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "pod": "ds-idrepo-2" 19:30:52 DEBUG }, 19:30:52 DEBUG "pod": "ds-cts-2" 19:30:52 DEBUG }, 19:30:52 DEBUG 19:30:52 DEBUG "3527703370752" 19:30:52 DEBUG }, 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "0.17293333333338978" 19:30:52 DEBUG }, 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG "0" 19:30:52 DEBUG ] 19:30:52 DEBUG 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "0" 19:30:52 DEBUG [...] 19:30:52 DEBUG ] 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG [...] 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG } 19:30:52 DEBUG 19:30:52 DEBUG } 19:30:52 DEBUG 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG "0" 19:30:52 DEBUG 19:30:52 DEBUG } 19:30:52 DEBUG 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:30:52 DEBUG }, 19:30:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:52 DEBUG 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG [...] 19:30:52 DEBUG "80881652.53333333" 19:30:52 DEBUG ] 19:30:52 DEBUG 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG ] 19:30:52 DEBUG 19:30:52 DEBUG ] 19:30:52 DEBUG ] 19:30:52 DEBUG [...] 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG "2670.5666666666666" 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG [...] 19:30:52 DEBUG 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG 19:30:52 DEBUG "0" 19:30:52 DEBUG 19:30:52 DEBUG } 19:30:52 DEBUG 19:30:52 DEBUG "pod": "ds-cts-2" 19:30:52 DEBUG }, 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG "0" 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG 19:30:52 DEBUG } 19:30:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:52 DEBUG }, 19:30:52 DEBUG [...] 19:30:52 DEBUG 19:30:52 DEBUG } 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG 19:30:52 DEBUG 19:30:52 DEBUG ] 19:30:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:30:52 DEBUG } 19:30:52 DEBUG "4.751548547310268" 19:30:52 DEBUG } 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG "0" 19:30:52 DEBUG 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG "0.0004" 19:30:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:52 DEBUG 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 19:30:52 DEBUG "pod": "ds-cts-2" 19:30:52 DEBUG }, 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG "480528" 19:30:52 DEBUG ] 19:30:52 DEBUG 19:30:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:30:52 DEBUG }, 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG }, 19:30:52 DEBUG 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG }, 19:30:52 DEBUG } 19:30:52 DEBUG "value": [ 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG "value": [ 19:30:52 DEBUG [...] 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG "value": [ 19:30:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:30:52 DEBUG "29857.967907174196" 19:30:52 DEBUG ] 19:30:52 DEBUG ] 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG "839090176" 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG }, 19:30:52 DEBUG "value": [ 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG 19:30:52 DEBUG "2670.8333333333335" 19:30:52 DEBUG "value": [ 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG ] 19:30:52 DEBUG "8.916420566436978" 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG 19:30:52 DEBUG ] 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG "30" 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG 19:30:52 DEBUG } 19:30:52 DEBUG "0.152" 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG "instance": "10.142.0.43:9100" 19:30:52 DEBUG ] 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG }, 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG "value": [ 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG 1713637851, 19:30:52 DEBUG } 19:30:52 DEBUG } 19:30:52 DEBUG "80883249.13333333" 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG ] 19:30:52 DEBUG } 19:30:52 DEBUG } 19:31:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637910" 19:31:50 ERROR [http_cmd]: ERROR 19:31:50 ERROR --- status code --- 19:31:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:31:50 ERROR --- http response --- 19:31:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:31:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:31:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637910" 19:31:50 INFO [http_cmd]: http status code OK 19:31:50 DEBUG --- status code --- 19:31:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:50 DEBUG --- http response --- 19:31:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:31:51 INFO 19:31:51 INFO [loop_until]: kubectl --namespace=xlou top pods 19:31:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:31:51 INFO [loop_until]: OK (rc = 0) 19:31:51 DEBUG --- stdout --- 19:31:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17078m 8652Mi am-767f6b67b5-qcqz2 16951m 9059Mi am-767f6b67b5-vbltz 18570m 8665Mi ds-cts-0 18696m 10218Mi ds-cts-1 17424m 10145Mi ds-cts-2 18861m 10200Mi ds-idrepo-0 7190m 3892Mi ds-idrepo-1 7250m 3908Mi ds-idrepo-2 7612m 3754Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 20m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8797m 708Mi 19:31:51 DEBUG --- stderr --- 19:31:51 DEBUG 19:31:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637911" 19:31:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637911" 19:31:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637911" 19:31:51 INFO [http_cmd]: http status code OK 19:31:51 INFO [http_cmd]: http status code OK 19:31:51 DEBUG --- status code --- 19:31:51 INFO [http_cmd]: http status code OK 19:31:51 DEBUG --- status code --- 19:31:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:51 DEBUG --- status code --- 19:31:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:51 DEBUG --- http response --- 19:31:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:51 DEBUG --- http response --- 19:31:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:51 DEBUG --- http response --- 19:31:51 DEBUG ----- output ----- 19:31:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:51 DEBUG { 19:31:51 DEBUG ----- output ----- 19:31:51 DEBUG ----- output ----- 19:31:51 DEBUG { 19:31:51 DEBUG { 19:31:51 DEBUG "status": "success", 19:31:51 DEBUG "status": "success", 19:31:51 DEBUG "status": "success", 19:31:51 DEBUG "data": { 19:31:51 DEBUG "data": { 19:31:51 DEBUG "data": { 19:31:51 DEBUG "resultType": "vector", 19:31:51 DEBUG "resultType": "vector", 19:31:51 DEBUG "resultType": "vector", 19:31:51 DEBUG "result": [ 19:31:51 DEBUG "result": [ 19:31:51 DEBUG "result": [ 19:31:51 DEBUG { 19:31:51 DEBUG { 19:31:51 DEBUG { 19:31:51 DEBUG "metric": { 19:31:51 DEBUG "metric": { 19:31:51 DEBUG "metric": { 19:31:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:51 DEBUG "pod": "ds-idrepo-0" 19:31:51 DEBUG "pod": "ds-idrepo-0" 19:31:51 DEBUG }, 19:31:51 DEBUG }, 19:31:51 DEBUG }, 19:31:51 DEBUG "value": [ 19:31:51 DEBUG "value": [ 19:31:51 DEBUG "value": [ 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG [...] 19:31:51 DEBUG [...] 19:31:51 DEBUG [...] 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG 19:31:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:51 DEBUG "pod": "ds-idrepo-2" 19:31:51 DEBUG "pod": "ds-idrepo-2" 19:31:51 DEBUG }, 19:31:51 DEBUG }, 19:31:51 DEBUG }, 19:31:51 DEBUG "value": [ 19:31:51 DEBUG "value": [ 19:31:51 DEBUG "value": [ 19:31:51 DEBUG 1713637911, 19:31:51 DEBUG 1713637911, 19:31:51 DEBUG 1713637911, 19:31:51 DEBUG "2672.133333333333" 19:31:51 DEBUG "0" 19:31:51 DEBUG "0" 19:31:51 DEBUG ] 19:31:51 DEBUG ] 19:31:51 DEBUG ] 19:31:51 DEBUG } 19:31:51 DEBUG } 19:31:51 DEBUG } 19:31:51 DEBUG ] 19:31:51 DEBUG } 19:31:51 DEBUG ] 19:31:51 DEBUG } 19:31:51 DEBUG } 19:31:51 DEBUG ] 19:31:51 DEBUG } 19:31:51 DEBUG } 19:31:51 DEBUG } 19:31:51 INFO 19:31:51 INFO [loop_until]: kubectl --namespace=xlou top node 19:31:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:31:51 INFO [loop_until]: OK (rc = 0) 19:31:51 DEBUG --- stdout --- 19:31:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6206Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7563m 25% 7625Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16976m 56% 11363Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 91m 0% 3346Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18719m 62% 13475Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5208Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17259m 57% 11950Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14561m 48% 2957Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19527m 65% 10890Mi 9% gke-xlou-cdm-ds-0f601751-j488 8216m 27% 7146Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17472m 58% 15266Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8216m 27% 7873Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19369m 64% 12928Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9096m 57% 6449Mi 10% 19:31:51 DEBUG --- stderr --- 19:31:51 DEBUG 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 DEBUG --- status code --- 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637912" 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637912" 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637912" 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 DEBUG --- status code --- 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 DEBUG --- status code --- 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG --- status code --- 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 DEBUG --- status code --- 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG --- status code --- 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 DEBUG --- http response --- 19:31:52 INFO [http_cmd]: http status code OK 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG --- status code --- 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:52 DEBUG { 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG { 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG --- http response --- 19:31:52 DEBUG { 19:31:52 DEBUG { 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:52 DEBUG { 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG { 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG { 19:31:52 DEBUG "data": { 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:31:52 DEBUG { 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG { 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:31:52 DEBUG { 19:31:52 DEBUG { 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "data": { 19:31:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG { 19:31:52 DEBUG { 19:31:52 DEBUG { 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG { 19:31:52 DEBUG { 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG { 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG { 19:31:52 DEBUG ----- output ----- 19:31:52 DEBUG "data": { 19:31:52 DEBUG "data": { 19:31:52 DEBUG { 19:31:52 DEBUG "data": { 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "data": { 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "data": { 19:31:52 DEBUG { 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "data": { 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "data": { 19:31:52 DEBUG { 19:31:52 DEBUG "data": { 19:31:52 DEBUG "data": { 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG { 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "data": { 19:31:52 DEBUG "data": { 19:31:52 DEBUG "data": { 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "data": { 19:31:52 DEBUG "data": { 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG { 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "data": { 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "result": [ 19:31:52 DEBUG { 19:31:52 DEBUG "data": { 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "result": [ 19:31:52 DEBUG { 19:31:52 DEBUG "data": { 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "status": "success", 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "data": { 19:31:52 DEBUG { 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "data": { 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG { 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "result": [ 19:31:52 DEBUG { 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "data": { 19:31:52 DEBUG { 19:31:52 DEBUG { 19:31:52 DEBUG "instance": "10.142.0.107:9100" 19:31:52 DEBUG { 19:31:52 DEBUG "metric": { 19:31:52 DEBUG { 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG { 19:31:52 DEBUG "pod": "ds-idrepo-0" 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "metric": { 19:31:52 DEBUG { 19:31:52 DEBUG { 19:31:52 DEBUG { 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:52 DEBUG "resultType": "vector", 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "metric": { 19:31:52 DEBUG }, 19:31:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "instance": "10.142.0.107:9100" 19:31:52 DEBUG "instance": "10.142.0.107:9100" 19:31:52 DEBUG { 19:31:52 DEBUG { 19:31:52 DEBUG { 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "metric": { 19:31:52 DEBUG }, 19:31:52 DEBUG { 19:31:52 DEBUG "instance": "10.142.0.107:9100" 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "metric": { 19:31:52 DEBUG { 19:31:52 DEBUG }, 19:31:52 DEBUG "result": [ 19:31:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:31:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:52 DEBUG { 19:31:52 DEBUG }, 19:31:52 DEBUG }, 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "metric": { 19:31:52 DEBUG { 19:31:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "metric": { 19:31:52 DEBUG }, 19:31:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:31:52 DEBUG "pod": "ds-idrepo-0" 19:31:52 DEBUG "pod": "ds-cts-0" 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "value": [ 19:31:52 DEBUG { 19:31:52 DEBUG }, 19:31:52 DEBUG }, 19:31:52 DEBUG 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:52 DEBUG "instance": "10.142.0.107:9100" 19:31:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:52 DEBUG "metric": { 19:31:52 DEBUG }, 19:31:52 DEBUG 19:31:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG }, 19:31:52 DEBUG }, 19:31:52 DEBUG "pod": "ds-cts-0" 19:31:52 DEBUG 19:31:52 DEBUG "metric": { 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "value": [ 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG }, 19:31:52 DEBUG }, 19:31:52 DEBUG }, 19:31:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:31:52 DEBUG "value": [ 19:31:52 DEBUG 19:31:52 DEBUG }, 19:31:52 DEBUG 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG 19:31:52 DEBUG "pod": "ds-cts-0" 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG }, 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG "value": [ 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG "value": [ 19:31:52 DEBUG 19:31:52 DEBUG }, 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG "value": [ 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG "value": [ 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG "value": [ 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG [...] 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG "instance": "10.142.0.43:9100" 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG [...] 19:31:52 DEBUG [...] 19:31:52 DEBUG "instance": "10.142.0.43:9100" 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:52 DEBUG [...] 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG }, 19:31:52 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG }, 19:31:52 DEBUG "instance": "10.142.0.43:9100" 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG "instance": "10.142.0.43:9100" 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG }, 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:31:52 DEBUG "instance": "10.142.0.43:9100" 19:31:52 DEBUG 19:31:52 DEBUG 19:31:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:52 DEBUG 19:31:52 DEBUG }, 19:31:52 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:31:52 DEBUG "pod": "ds-idrepo-2" 19:31:52 DEBUG "pod": "ds-cts-2" 19:31:52 DEBUG 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:31:52 DEBUG "pod": "ds-cts-2" 19:31:52 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:52 DEBUG }, 19:31:52 DEBUG }, 19:31:52 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:31:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:31:52 DEBUG }, 19:31:52 DEBUG "pod": "ds-cts-2" 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG }, 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "pod": "ds-cts-2" 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG }, 19:31:52 DEBUG }, 19:31:52 DEBUG }, 19:31:52 DEBUG "0.0683333333333356" 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG }, 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG "value": [ 19:31:52 DEBUG }, 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG }, 19:31:52 DEBUG "2660.6" 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "value": [ 19:31:52 DEBUG ] 19:31:52 DEBUG "5.628226615440324" 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "0.18306666666676677" 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG "value": [ 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "value": [ 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "3531851682816" 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG "value": [ 19:31:52 DEBUG "0" 19:31:52 DEBUG "value": [ 19:31:52 DEBUG ] 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG "2660.2666666666673" 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG ] 19:31:52 DEBUG "80682879.1" 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG "841682944" 19:31:52 DEBUG "80687548.43333334" 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG "0" 19:31:52 DEBUG "0" 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG ] 19:31:52 DEBUG "16286.824331863087" 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG ] 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG } 19:31:52 DEBUG "8.892939554968184" 19:31:52 DEBUG "27.366666666666667" 19:31:52 DEBUG "0" 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG "0.152" 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG "480534" 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG 1713637912, 19:31:52 DEBUG ] 19:31:52 DEBUG "0" 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG "0.0005333333333333334" 19:31:52 DEBUG } 19:31:52 DEBUG "0" 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG "2660.7666666666673" 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG ] 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:31:52 DEBUG } 19:32:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713637970" 19:32:50 ERROR [http_cmd]: ERROR 19:32:50 ERROR --- status code --- 19:32:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:32:50 ERROR --- http response --- 19:32:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:32:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:32:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637970" 19:32:50 INFO [http_cmd]: http status code OK 19:32:50 DEBUG --- status code --- 19:32:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:50 DEBUG --- http response --- 19:32:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:32:51 INFO 19:32:51 INFO [loop_until]: kubectl --namespace=xlou top pods 19:32:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:32:51 INFO [loop_until]: OK (rc = 0) 19:32:51 DEBUG --- stdout --- 19:32:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17278m 8651Mi am-767f6b67b5-qcqz2 16931m 9060Mi am-767f6b67b5-vbltz 18501m 8664Mi ds-cts-0 18982m 10215Mi ds-cts-1 17242m 10160Mi ds-cts-2 18856m 10212Mi ds-idrepo-0 7325m 3893Mi ds-idrepo-1 7345m 3907Mi ds-idrepo-2 7453m 3754Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8948m 711Mi 19:32:51 DEBUG --- stderr --- 19:32:51 DEBUG 19:32:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637971" 19:32:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637971" 19:32:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637971" 19:32:51 INFO [http_cmd]: http status code OK 19:32:51 INFO [http_cmd]: http status code OK 19:32:51 DEBUG --- status code --- 19:32:51 DEBUG --- status code --- 19:32:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:51 DEBUG --- http response --- 19:32:51 DEBUG --- http response --- 19:32:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:51 DEBUG ----- output ----- 19:32:51 DEBUG ----- output ----- 19:32:51 DEBUG { 19:32:51 DEBUG { 19:32:51 DEBUG "status": "success", 19:32:51 DEBUG "status": "success", 19:32:51 DEBUG "data": { 19:32:51 INFO [http_cmd]: http status code OK 19:32:51 DEBUG "data": { 19:32:51 DEBUG "resultType": "vector", 19:32:51 DEBUG --- status code --- 19:32:51 DEBUG "resultType": "vector", 19:32:51 DEBUG "result": [ 19:32:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:51 DEBUG "result": [ 19:32:51 DEBUG { 19:32:51 DEBUG --- http response --- 19:32:51 DEBUG { 19:32:51 DEBUG "metric": { 19:32:51 DEBUG "metric": { 19:32:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:51 DEBUG "pod": "ds-idrepo-0" 19:32:51 DEBUG "pod": "ds-idrepo-0" 19:32:51 DEBUG ----- output ----- 19:32:51 DEBUG }, 19:32:51 DEBUG }, 19:32:51 DEBUG { 19:32:51 DEBUG "value": [ 19:32:51 DEBUG "value": [ 19:32:51 DEBUG "status": "success", 19:32:51 DEBUG 19:32:51 DEBUG 19:32:51 DEBUG "data": { 19:32:51 DEBUG 19:32:51 DEBUG 19:32:51 DEBUG "resultType": "vector", 19:32:51 DEBUG 19:32:51 DEBUG 19:32:51 DEBUG "result": [ 19:32:51 DEBUG [...] 19:32:51 DEBUG [...] 19:32:51 DEBUG { 19:32:51 DEBUG 19:32:51 DEBUG 19:32:51 DEBUG "metric": { 19:32:51 DEBUG 19:32:51 DEBUG 19:32:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:51 DEBUG 19:32:51 DEBUG 19:32:51 DEBUG }, 19:32:51 DEBUG "pod": "ds-idrepo-2" 19:32:51 DEBUG "pod": "ds-idrepo-2" 19:32:51 DEBUG "value": [ 19:32:51 DEBUG }, 19:32:51 DEBUG }, 19:32:51 DEBUG 19:32:51 DEBUG "value": [ 19:32:51 DEBUG "value": [ 19:32:51 DEBUG 19:32:51 DEBUG 1713637971, 19:32:51 DEBUG 1713637971, 19:32:51 DEBUG 19:32:51 DEBUG "0" 19:32:51 DEBUG "0" 19:32:51 DEBUG [...] 19:32:51 DEBUG ] 19:32:51 DEBUG ] 19:32:51 DEBUG 19:32:51 DEBUG } 19:32:51 DEBUG } 19:32:51 DEBUG 19:32:51 DEBUG ] 19:32:51 DEBUG ] 19:32:51 DEBUG 19:32:51 DEBUG } 19:32:51 DEBUG } 19:32:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:51 DEBUG } 19:32:51 DEBUG } 19:32:51 DEBUG }, 19:32:51 DEBUG "value": [ 19:32:51 DEBUG 1713637971, 19:32:51 DEBUG "2670.8329167083293" 19:32:51 DEBUG ] 19:32:51 DEBUG } 19:32:51 DEBUG ] 19:32:51 DEBUG } 19:32:51 DEBUG } 19:32:51 INFO 19:32:51 INFO [loop_until]: kubectl --namespace=xlou top node 19:32:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:32:52 INFO [loop_until]: OK (rc = 0) 19:32:52 DEBUG --- stdout --- 19:32:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6216Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7558m 25% 7651Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16912m 56% 11373Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3336Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19311m 64% 13478Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 141m 0% 5218Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17287m 57% 11952Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14582m 48% 2965Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19473m 65% 10888Mi 9% gke-xlou-cdm-ds-0f601751-j488 8210m 27% 7157Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17372m 58% 15249Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8177m 27% 7864Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18367m 61% 12924Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9080m 57% 6448Mi 10% 19:32:52 DEBUG --- stderr --- 19:32:52 DEBUG 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713637972" 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 DEBUG --- status code --- 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 INFO [http_cmd]: http status code OK 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG --- status code --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:32:52 DEBUG --- http response --- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:32:52 DEBUG { 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG { 19:32:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG { 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "data": { 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG ----- output ----- 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "data": { 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG { 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG { 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG { 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "data": { 19:32:52 DEBUG { 19:32:52 DEBUG "data": { 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "data": { 19:32:52 DEBUG "data": { 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "data": { 19:32:52 DEBUG "data": { 19:32:52 DEBUG "data": { 19:32:52 DEBUG "data": { 19:32:52 DEBUG "data": { 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "data": { 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "data": { 19:32:52 DEBUG "data": { 19:32:52 DEBUG "data": { 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "data": { 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "data": { 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "status": "success", 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "data": { 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "data": { 19:32:52 DEBUG { 19:32:52 DEBUG "data": { 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "data": { 19:32:52 DEBUG "result": [ 19:32:52 DEBUG { 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG { 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "data": { 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "metric": { 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "result": [ 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG { 19:32:52 DEBUG "metric": { 19:32:52 DEBUG "result": [ 19:32:52 DEBUG { 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "metric": { 19:32:52 DEBUG { 19:32:52 DEBUG "resultType": "vector", 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG { 19:32:52 DEBUG "result": [ 19:32:52 DEBUG { 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "instance": "10.142.0.107:9100" 19:32:52 DEBUG "result": [ 19:32:52 DEBUG { 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "metric": { 19:32:52 DEBUG "metric": { 19:32:52 DEBUG { 19:32:52 DEBUG "result": [ 19:32:52 DEBUG "metric": { 19:32:52 DEBUG "instance": "10.142.0.107:9100" 19:32:52 DEBUG { 19:32:52 DEBUG "metric": { 19:32:52 DEBUG { 19:32:52 DEBUG "instance": "10.142.0.107:9100" 19:32:52 DEBUG "metric": { 19:32:52 DEBUG "result": [ 19:32:52 DEBUG { 19:32:52 DEBUG "metric": { 19:32:52 DEBUG "metric": { 19:32:52 DEBUG { 19:32:52 DEBUG "metric": { 19:32:52 DEBUG { 19:32:52 DEBUG }, 19:32:52 DEBUG { 19:32:52 DEBUG "metric": { 19:32:52 DEBUG { 19:32:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:52 DEBUG "pod": "ds-cts-0" 19:32:52 DEBUG "metric": { 19:32:52 DEBUG { 19:32:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:52 DEBUG }, 19:32:52 DEBUG "metric": { 19:32:52 DEBUG "instance": "10.142.0.107:9100" 19:32:52 DEBUG "metric": { 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "ds-idrepo-0" 19:32:52 DEBUG "metric": { 19:32:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:32:52 DEBUG "metric": { 19:32:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:32:52 DEBUG "metric": { 19:32:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:32:52 DEBUG "metric": { 19:32:52 DEBUG "pod": "ds-cts-0" 19:32:52 DEBUG "metric": { 19:32:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:52 DEBUG "metric": { 19:32:52 DEBUG }, 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:32:52 DEBUG "metric": { 19:32:52 DEBUG }, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "pod": "ds-idrepo-0" 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "ds-cts-0" 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:52 DEBUG }, 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:32:52 DEBUG }, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:32:52 DEBUG }, 19:32:52 DEBUG "instance": "10.142.0.107:9100" 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 19:32:52 DEBUG }, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 19:32:52 DEBUG }, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG [...] 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "instance": "10.142.0.43:9100" 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "instance": "10.142.0.43:9100" 19:32:52 DEBUG "instance": "10.142.0.43:9100" 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "instance": "10.142.0.43:9100" 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:52 DEBUG "pod": "ds-cts-2" 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:52 DEBUG }, 19:32:52 DEBUG 19:32:52 DEBUG 19:32:52 DEBUG }, 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "ds-cts-2" 19:32:52 DEBUG 19:32:52 DEBUG }, 19:32:52 DEBUG 19:32:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:32:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:32:52 DEBUG 19:32:52 DEBUG "pod": "ds-cts-2" 19:32:52 DEBUG }, 19:32:52 DEBUG 19:32:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:52 DEBUG 19:32:52 DEBUG }, 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:32:52 DEBUG 19:32:52 DEBUG }, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "pod": "ds-idrepo-2" 19:32:52 DEBUG 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 19:32:52 DEBUG }, 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:32:52 DEBUG }, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG }, 19:32:52 DEBUG "instance": "10.142.0.43:9100" 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG }, 19:32:52 DEBUG "pod": "ds-cts-2" 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:32:52 DEBUG "80805179.36666667" 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG "0.09266666666667903" 19:32:52 DEBUG "value": [ 19:32:52 DEBUG }, 19:32:52 DEBUG "0.2335666666666166" 19:32:52 DEBUG "3536152826880" 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "0" 19:32:52 DEBUG }, 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG }, 19:32:52 DEBUG ] 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "0.152" 19:32:52 DEBUG "29.933333333333334" 19:32:52 DEBUG ] 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "2657.365736573657" 19:32:52 DEBUG ] 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG "value": [ 19:32:52 DEBUG ] 19:32:52 DEBUG ] 19:32:52 DEBUG "0" 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG ] 19:32:52 DEBUG "value": [ 19:32:52 DEBUG "842997760" 19:32:52 DEBUG "8.867491614239107" 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG "0.0007" 19:32:52 DEBUG "value": [ 19:32:52 DEBUG } 19:32:52 DEBUG "2655.4655465546557" 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG ] 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG } 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG "0" 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG "2655.8989232256563" 19:32:52 DEBUG } 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG ] 19:32:52 DEBUG ] 19:32:52 DEBUG "0" 19:32:52 DEBUG ] 19:32:52 DEBUG 1713637972, 19:32:52 DEBUG ] 19:32:52 DEBUG ] 19:32:52 DEBUG "80809633.69999999" 19:32:52 DEBUG } 19:32:52 DEBUG "0" 19:32:52 DEBUG ] 19:32:52 DEBUG "0" 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG ] 19:32:52 DEBUG "0" 19:32:52 DEBUG ] 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG ] 19:32:52 DEBUG "479792" 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG "914.4750599823211" 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG ] 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG ] 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG ] 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:32:52 DEBUG } 19:33:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638030" 19:33:50 ERROR [http_cmd]: ERROR 19:33:50 ERROR --- status code --- 19:33:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:33:50 ERROR --- http response --- 19:33:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:33:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:33:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638030" 19:33:50 INFO [http_cmd]: http status code OK 19:33:50 DEBUG --- status code --- 19:33:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:50 DEBUG --- http response --- 19:33:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:33:51 INFO 19:33:51 INFO [loop_until]: kubectl --namespace=xlou top pods 19:33:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:33:51 INFO [loop_until]: OK (rc = 0) 19:33:51 DEBUG --- stdout --- 19:33:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17450m 8651Mi am-767f6b67b5-qcqz2 17068m 9060Mi am-767f6b67b5-vbltz 18240m 8664Mi ds-cts-0 18898m 10201Mi ds-cts-1 17533m 10162Mi ds-cts-2 18954m 10213Mi ds-idrepo-0 7276m 3893Mi ds-idrepo-1 7356m 3907Mi ds-idrepo-2 7444m 3754Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 14m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9013m 712Mi 19:33:51 DEBUG --- stderr --- 19:33:51 DEBUG 19:33:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638031" 19:33:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638031" 19:33:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638031" 19:33:51 INFO [http_cmd]: http status code OK 19:33:51 DEBUG --- status code --- 19:33:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:51 DEBUG --- http response --- 19:33:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:51 INFO [http_cmd]: http status code OK 19:33:51 INFO [http_cmd]: http status code OK 19:33:51 DEBUG ----- output ----- 19:33:51 DEBUG --- status code --- 19:33:51 DEBUG --- status code --- 19:33:51 DEBUG { 19:33:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:51 DEBUG "status": "success", 19:33:51 DEBUG --- http response --- 19:33:51 DEBUG --- http response --- 19:33:51 DEBUG "data": { 19:33:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:51 DEBUG "resultType": "vector", 19:33:51 DEBUG ----- output ----- 19:33:51 DEBUG ----- output ----- 19:33:51 DEBUG "result": [ 19:33:51 DEBUG { 19:33:51 DEBUG { 19:33:51 DEBUG { 19:33:51 DEBUG "status": "success", 19:33:51 DEBUG "status": "success", 19:33:51 DEBUG "metric": { 19:33:51 DEBUG "data": { 19:33:51 DEBUG "resultType": "vector", 19:33:51 DEBUG "data": { 19:33:51 DEBUG "pod": "ds-idrepo-0" 19:33:51 DEBUG "result": [ 19:33:51 DEBUG "resultType": "vector", 19:33:51 DEBUG }, 19:33:51 DEBUG { 19:33:51 DEBUG "result": [ 19:33:51 DEBUG "value": [ 19:33:51 DEBUG "metric": { 19:33:51 DEBUG { 19:33:51 DEBUG 19:33:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:51 DEBUG "metric": { 19:33:51 DEBUG 19:33:51 DEBUG }, 19:33:51 DEBUG "pod": "ds-idrepo-0" 19:33:51 DEBUG 19:33:51 DEBUG "value": [ 19:33:51 DEBUG }, 19:33:51 DEBUG [...] 19:33:51 DEBUG 19:33:51 DEBUG "value": [ 19:33:51 DEBUG 19:33:51 DEBUG 19:33:51 DEBUG 19:33:51 DEBUG 19:33:51 DEBUG 19:33:51 DEBUG 19:33:51 DEBUG 19:33:51 DEBUG [...] 19:33:51 DEBUG [...] 19:33:51 DEBUG 19:33:51 DEBUG 19:33:51 DEBUG 19:33:51 DEBUG 19:33:51 DEBUG 19:33:51 DEBUG 19:33:51 DEBUG "pod": "ds-idrepo-2" 19:33:51 DEBUG 19:33:51 DEBUG "pod": "ds-idrepo-2" 19:33:51 DEBUG }, 19:33:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:51 DEBUG }, 19:33:51 DEBUG "value": [ 19:33:51 DEBUG }, 19:33:51 DEBUG "value": [ 19:33:51 DEBUG 1713638031, 19:33:51 DEBUG "value": [ 19:33:51 DEBUG 1713638031, 19:33:51 DEBUG "0" 19:33:51 DEBUG 1713638031, 19:33:51 DEBUG "0" 19:33:51 DEBUG ] 19:33:51 DEBUG "2673.6666666666665" 19:33:51 DEBUG ] 19:33:51 DEBUG } 19:33:51 DEBUG ] 19:33:51 DEBUG } 19:33:51 DEBUG ] 19:33:51 DEBUG } 19:33:51 DEBUG } 19:33:51 DEBUG ] 19:33:51 DEBUG ] 19:33:51 DEBUG } 19:33:51 DEBUG } 19:33:51 DEBUG } 19:33:51 DEBUG } 19:33:51 DEBUG } 19:33:52 INFO 19:33:52 INFO [loop_until]: kubectl --namespace=xlou top node 19:33:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:33:52 INFO [loop_until]: OK (rc = 0) 19:33:52 DEBUG --- stdout --- 19:33:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6219Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7563m 25% 7631Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16389m 54% 11358Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3339Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18725m 62% 13472Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5208Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17218m 57% 11950Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14543m 48% 2951Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19304m 64% 10878Mi 9% gke-xlou-cdm-ds-0f601751-j488 8111m 27% 7160Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16865m 56% 15276Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8106m 27% 7871Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18920m 63% 12941Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9084m 57% 6444Mi 10% 19:33:52 DEBUG --- stderr --- 19:33:52 DEBUG 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638032" 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG { 19:33:52 DEBUG { 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "data": { 19:33:52 DEBUG "data": { 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "result": [ 19:33:52 DEBUG { 19:33:52 DEBUG { 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:33:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:33:52 DEBUG }, 19:33:52 DEBUG }, 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638032" 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 DEBUG 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638032" 19:33:52 DEBUG 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638032" 19:33:52 DEBUG [...] 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 DEBUG [...] 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638032" 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 DEBUG 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 DEBUG --- status code --- 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG --- status code --- 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 DEBUG 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG --- http response --- 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 INFO [http_cmd]: http status code OK 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG }, 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:52 DEBUG }, 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG --- status code --- 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG "8.8663339734022" 19:33:52 DEBUG ] 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG "value": [ 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG } 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG { 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:33:52 DEBUG { 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:33:52 DEBUG { 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG { 19:33:52 DEBUG { 19:33:52 DEBUG { 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:52 DEBUG { 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG { 19:33:52 DEBUG { 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "1753.1105394612066" 19:33:52 DEBUG --- http response --- 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:33:52 DEBUG { 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:33:52 DEBUG } 19:33:52 DEBUG { 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG { 19:33:52 DEBUG { 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG { 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG "data": { 19:33:52 DEBUG ] 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG { 19:33:52 DEBUG ----- output ----- 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG { 19:33:52 DEBUG "data": { 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "data": { 19:33:52 DEBUG "data": { 19:33:52 DEBUG "data": { 19:33:52 DEBUG { 19:33:52 DEBUG "data": { 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "data": { 19:33:52 DEBUG "data": { 19:33:52 DEBUG { 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG "data": { 19:33:52 DEBUG "data": { 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG { 19:33:52 DEBUG "data": { 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "data": { 19:33:52 DEBUG "data": { 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "data": { 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "result": [ 19:33:52 DEBUG } 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG { 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "data": { 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "data": { 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "data": { 19:33:52 DEBUG { 19:33:52 DEBUG } 19:33:52 DEBUG "result": [ 19:33:52 DEBUG { 19:33:52 DEBUG "data": { 19:33:52 DEBUG "data": { 19:33:52 DEBUG "status": "success", 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG { 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "result": [ 19:33:52 DEBUG { 19:33:52 DEBUG { 19:33:52 DEBUG { 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG { 19:33:52 DEBUG "result": [ 19:33:52 DEBUG { 19:33:52 DEBUG { 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "metric": { 19:33:52 DEBUG { 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "data": { 19:33:52 DEBUG { 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "metric": { 19:33:52 DEBUG { 19:33:52 DEBUG { 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "metric": { 19:33:52 DEBUG { 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "resultType": "vector", 19:33:52 DEBUG "metric": { 19:33:52 DEBUG { 19:33:52 DEBUG "instance": "10.142.0.107:9100" 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "pod": "ds-cts-0" 19:33:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:52 DEBUG "instance": "10.142.0.107:9100" 19:33:52 DEBUG { 19:33:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:33:52 DEBUG "instance": "10.142.0.107:9100" 19:33:52 DEBUG { 19:33:52 DEBUG }, 19:33:52 DEBUG "pod": "ds-idrepo-0" 19:33:52 DEBUG }, 19:33:52 DEBUG { 19:33:52 DEBUG { 19:33:52 DEBUG "result": [ 19:33:52 DEBUG "instance": "10.142.0.107:9100" 19:33:52 DEBUG "metric": { 19:33:52 DEBUG }, 19:33:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:33:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:52 DEBUG }, 19:33:52 DEBUG }, 19:33:52 DEBUG }, 19:33:52 DEBUG "metric": { 19:33:52 DEBUG }, 19:33:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:33:52 DEBUG }, 19:33:52 DEBUG }, 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "value": [ 19:33:52 DEBUG }, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "metric": { 19:33:52 DEBUG { 19:33:52 DEBUG }, 19:33:52 DEBUG "pod": "ds-cts-0" 19:33:52 DEBUG "value": [ 19:33:52 DEBUG }, 19:33:52 DEBUG }, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:33:52 DEBUG "value": [ 19:33:52 DEBUG }, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "pod": "ds-cts-0" 19:33:52 DEBUG 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 19:33:52 DEBUG "pod": "ds-idrepo-0" 19:33:52 DEBUG "instance": "10.142.0.107:9100" 19:33:52 DEBUG "metric": { 19:33:52 DEBUG "value": [ 19:33:52 DEBUG }, 19:33:52 DEBUG 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG }, 19:33:52 DEBUG 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG }, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG }, 19:33:52 DEBUG }, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG }, 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG [...] 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG [...] 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG "pod": "ds-cts-2" 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:52 DEBUG }, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG "instance": "10.142.0.43:9100" 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG }, 19:33:52 DEBUG 19:33:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:52 DEBUG 19:33:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:52 DEBUG 19:33:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:52 DEBUG "instance": "10.142.0.43:9100" 19:33:52 DEBUG 19:33:52 DEBUG "pod": "ds-cts-2" 19:33:52 DEBUG "pod": "ds-cts-2" 19:33:52 DEBUG }, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 19:33:52 DEBUG "2666.8000000000006" 19:33:52 DEBUG 19:33:52 DEBUG "instance": "10.142.0.43:9100" 19:33:52 DEBUG }, 19:33:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:33:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:52 DEBUG "value": [ 19:33:52 DEBUG }, 19:33:52 DEBUG "instance": "10.142.0.43:9100" 19:33:52 DEBUG 19:33:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:33:52 DEBUG }, 19:33:52 DEBUG }, 19:33:52 DEBUG }, 19:33:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG ] 19:33:52 DEBUG 19:33:52 DEBUG 19:33:52 DEBUG "value": [ 19:33:52 DEBUG }, 19:33:52 DEBUG }, 19:33:52 DEBUG }, 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG }, 19:33:52 DEBUG }, 19:33:52 DEBUG }, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "value": [ 19:33:52 DEBUG }, 19:33:52 DEBUG "0" 19:33:52 DEBUG }, 19:33:52 DEBUG [...] 19:33:52 DEBUG "instance": "10.142.0.43:9100" 19:33:52 DEBUG } 19:33:52 DEBUG 19:33:52 DEBUG "pod": "ds-cts-2" 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "30" 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "value": [ 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG ] 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 19:33:52 DEBUG }, 19:33:52 DEBUG ] 19:33:52 DEBUG "pod": "ds-idrepo-2" 19:33:52 DEBUG }, 19:33:52 DEBUG "80816768.73333332" 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG ] 19:33:52 DEBUG "2666.0000000000005" 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG "0" 19:33:52 DEBUG "0.08899999999999104" 19:33:52 DEBUG "0.0004666666666666666" 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG } 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG 19:33:52 DEBUG "value": [ 19:33:52 DEBUG } 19:33:52 DEBUG }, 19:33:52 DEBUG "value": [ 19:33:52 DEBUG ] 19:33:52 DEBUG "3540135908352" 19:33:52 DEBUG "842985472" 19:33:52 DEBUG "0" 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG "0.228433333333291" 19:33:52 DEBUG "242694377.62671262" 19:33:52 DEBUG "11.4208931923427" 19:33:52 DEBUG ] 19:33:52 DEBUG ] 19:33:52 DEBUG ] 19:33:52 DEBUG "2665.733333333333" 19:33:52 DEBUG ] 19:33:52 DEBUG "0.1525" 19:33:52 DEBUG 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG } 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG ] 19:33:52 DEBUG ] 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG ] 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:33:52 DEBUG "80812224.03333333" 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG "0" 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG ] 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG }, 19:33:52 DEBUG ] 19:33:52 DEBUG "0" 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG ] 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG "value": [ 19:33:52 DEBUG 1713638032, 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG "479738" 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG ] 19:33:52 DEBUG } 19:33:52 DEBUG } 19:33:52 DEBUG } 19:34:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638090" 19:34:50 ERROR [http_cmd]: ERROR 19:34:50 ERROR --- status code --- 19:34:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:34:50 ERROR --- http response --- 19:34:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:34:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:34:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638090" 19:34:50 INFO [http_cmd]: http status code OK 19:34:50 DEBUG --- status code --- 19:34:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:50 DEBUG --- http response --- 19:34:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:34:51 INFO 19:34:51 INFO [loop_until]: kubectl --namespace=xlou top pods 19:34:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:34:51 INFO [loop_until]: OK (rc = 0) 19:34:51 DEBUG --- stdout --- 19:34:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16662m 8651Mi am-767f6b67b5-qcqz2 17101m 9060Mi am-767f6b67b5-vbltz 18749m 8666Mi ds-cts-0 19034m 10225Mi ds-cts-1 17333m 10172Mi ds-cts-2 18953m 10217Mi ds-idrepo-0 7312m 3896Mi ds-idrepo-1 7500m 3908Mi ds-idrepo-2 7438m 3757Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 20m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8938m 715Mi 19:34:51 DEBUG --- stderr --- 19:34:51 DEBUG 19:34:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638091" 19:34:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638091" 19:34:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638091" 19:34:51 INFO [http_cmd]: http status code OK 19:34:51 DEBUG --- status code --- 19:34:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:51 DEBUG --- http response --- 19:34:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:51 DEBUG ----- output ----- 19:34:51 DEBUG { 19:34:51 DEBUG "status": "success", 19:34:51 DEBUG "data": { 19:34:51 DEBUG "resultType": "vector", 19:34:51 DEBUG "result": [ 19:34:51 INFO [http_cmd]: http status code OK 19:34:51 DEBUG { 19:34:51 INFO [http_cmd]: http status code OK 19:34:51 DEBUG --- status code --- 19:34:51 DEBUG "metric": { 19:34:51 DEBUG --- status code --- 19:34:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:51 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:51 DEBUG --- http response --- 19:34:51 DEBUG }, 19:34:51 DEBUG --- http response --- 19:34:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:51 DEBUG "value": [ 19:34:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:51 DEBUG ----- output ----- 19:34:51 DEBUG 19:34:51 DEBUG ----- output ----- 19:34:51 DEBUG { 19:34:51 DEBUG 19:34:51 DEBUG { 19:34:51 DEBUG "status": "success", 19:34:51 DEBUG 19:34:51 DEBUG "status": "success", 19:34:51 DEBUG "data": { 19:34:51 DEBUG [...] 19:34:51 DEBUG "data": { 19:34:51 DEBUG "resultType": "vector", 19:34:51 DEBUG 19:34:51 DEBUG "resultType": "vector", 19:34:51 DEBUG "result": [ 19:34:51 DEBUG 19:34:51 DEBUG "result": [ 19:34:51 DEBUG { 19:34:51 DEBUG 19:34:51 DEBUG { 19:34:51 DEBUG "metric": { 19:34:51 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:51 DEBUG "metric": { 19:34:51 DEBUG "pod": "ds-idrepo-0" 19:34:51 DEBUG }, 19:34:51 DEBUG "pod": "ds-idrepo-0" 19:34:51 DEBUG }, 19:34:51 DEBUG "value": [ 19:34:51 DEBUG 1713638091, 19:34:51 DEBUG "value": [ 19:34:51 DEBUG "2674.4333333333334" 19:34:51 DEBUG }, 19:34:51 DEBUG 19:34:51 DEBUG ] 19:34:51 DEBUG "value": [ 19:34:51 DEBUG 19:34:51 DEBUG } 19:34:51 DEBUG 19:34:51 DEBUG 19:34:51 DEBUG ] 19:34:51 DEBUG 19:34:51 DEBUG [...] 19:34:51 DEBUG } 19:34:51 DEBUG 19:34:51 DEBUG 19:34:51 DEBUG } 19:34:51 DEBUG [...] 19:34:51 DEBUG 19:34:51 DEBUG 19:34:51 DEBUG 19:34:51 DEBUG 19:34:51 DEBUG 19:34:51 DEBUG "pod": "ds-idrepo-2" 19:34:51 DEBUG "pod": "ds-idrepo-2" 19:34:51 DEBUG }, 19:34:51 DEBUG }, 19:34:51 DEBUG "value": [ 19:34:51 DEBUG "value": [ 19:34:51 DEBUG 1713638091, 19:34:51 DEBUG 1713638091, 19:34:51 DEBUG "0" 19:34:51 DEBUG "0" 19:34:51 DEBUG ] 19:34:51 DEBUG ] 19:34:51 DEBUG } 19:34:51 DEBUG } 19:34:51 DEBUG ] 19:34:51 DEBUG ] 19:34:51 DEBUG } 19:34:51 DEBUG } 19:34:51 DEBUG } 19:34:51 DEBUG } 19:34:52 INFO 19:34:52 INFO [loop_until]: kubectl --namespace=xlou top node 19:34:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:34:52 INFO [loop_until]: OK (rc = 0) 19:34:52 DEBUG --- stdout --- 19:34:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6215Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7572m 25% 7634Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16946m 56% 11369Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3349Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18507m 61% 13478Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5199Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17230m 57% 11944Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14569m 48% 2972Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19491m 65% 10886Mi 9% gke-xlou-cdm-ds-0f601751-j488 8164m 27% 7158Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17209m 57% 15280Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8112m 27% 7873Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18913m 63% 12936Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8844m 55% 6445Mi 10% 19:34:52 DEBUG --- stderr --- 19:34:52 DEBUG 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 DEBUG --- status code --- 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG { 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "data": { 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "result": [ 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG { 19:34:52 DEBUG { 19:34:52 DEBUG "metric": { 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:34:52 DEBUG "data": { 19:34:52 DEBUG }, 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "result": [ 19:34:52 DEBUG 19:34:52 DEBUG { 19:34:52 DEBUG 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 DEBUG "metric": { 19:34:52 DEBUG 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:34:52 DEBUG [...] 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG }, 19:34:52 DEBUG 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:34:52 DEBUG 19:34:52 DEBUG { 19:34:52 DEBUG }, 19:34:52 DEBUG [...] 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 19:34:52 DEBUG "data": { 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "8.981713868136975" 19:34:52 DEBUG 19:34:52 DEBUG "result": [ 19:34:52 DEBUG ] 19:34:52 DEBUG "pod": "ds-cts-0" 19:34:52 DEBUG { 19:34:52 DEBUG } 19:34:52 DEBUG }, 19:34:52 DEBUG "metric": { 19:34:52 DEBUG ] 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:52 DEBUG } 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG }, 19:34:52 DEBUG } 19:34:52 DEBUG "35450092.84516376" 19:34:52 DEBUG "value": [ 19:34:52 DEBUG ] 19:34:52 DEBUG 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG 19:34:52 DEBUG } 19:34:52 DEBUG 19:34:52 DEBUG } 19:34:52 DEBUG [...] 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:52 DEBUG }, 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG "2664.233333333334" 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638092" 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638092" 19:34:52 DEBUG --- status code --- 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 DEBUG --- status code --- 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 DEBUG --- status code --- 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- status code --- 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 INFO [http_cmd]: http status code OK 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG { 19:34:52 DEBUG --- status code --- 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG { 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:52 DEBUG { 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG { 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:52 DEBUG { 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:52 DEBUG { 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG { 19:34:52 DEBUG { 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG "data": { 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG { 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "data": { 19:34:52 DEBUG { 19:34:52 DEBUG { 19:34:52 DEBUG "data": { 19:34:52 DEBUG { 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "data": { 19:34:52 DEBUG { 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG { 19:34:52 DEBUG { 19:34:52 DEBUG { 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG --- http response --- 19:34:52 DEBUG "data": { 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG { 19:34:52 DEBUG "data": { 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "data": { 19:34:52 DEBUG "data": { 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG { 19:34:52 DEBUG "data": { 19:34:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "result": [ 19:34:52 DEBUG { 19:34:52 DEBUG "data": { 19:34:52 DEBUG "data": { 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "data": { 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "data": { 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "data": { 19:34:52 DEBUG "data": { 19:34:52 DEBUG "data": { 19:34:52 DEBUG "metric": { 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG ----- output ----- 19:34:52 DEBUG "result": [ 19:34:52 DEBUG { 19:34:52 DEBUG "metric": { 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG { 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "data": { 19:34:52 DEBUG { 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG { 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "pod": "ds-idrepo-0" 19:34:52 DEBUG "result": [ 19:34:52 DEBUG { 19:34:52 DEBUG { 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:52 DEBUG "data": { 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "metric": { 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG "metric": { 19:34:52 DEBUG { 19:34:52 DEBUG "result": [ 19:34:52 DEBUG { 19:34:52 DEBUG "metric": { 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "result": [ 19:34:52 DEBUG }, 19:34:52 DEBUG { 19:34:52 DEBUG "status": "success", 19:34:52 DEBUG "metric": { 19:34:52 DEBUG "result": [ 19:34:52 DEBUG }, 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG { 19:34:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:52 DEBUG { 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "instance": "10.142.0.107:9100" 19:34:52 DEBUG "metric": { 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "metric": { 19:34:52 DEBUG "instance": "10.142.0.107:9100" 19:34:52 DEBUG { 19:34:52 DEBUG { 19:34:52 DEBUG { 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "metric": { 19:34:52 DEBUG "data": { 19:34:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:52 DEBUG { 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "metric": { 19:34:52 DEBUG }, 19:34:52 DEBUG "metric": { 19:34:52 DEBUG { 19:34:52 DEBUG }, 19:34:52 DEBUG "pod": "ds-cts-0" 19:34:52 DEBUG { 19:34:52 DEBUG "instance": "10.142.0.107:9100" 19:34:52 DEBUG }, 19:34:52 DEBUG "metric": { 19:34:52 DEBUG "metric": { 19:34:52 DEBUG "metric": { 19:34:52 DEBUG 19:34:52 DEBUG "instance": "10.142.0.107:9100" 19:34:52 DEBUG "resultType": "vector", 19:34:52 DEBUG }, 19:34:52 DEBUG "metric": { 19:34:52 DEBUG 19:34:52 DEBUG { 19:34:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:34:52 DEBUG "metric": { 19:34:52 DEBUG "value": [ 19:34:52 DEBUG }, 19:34:52 DEBUG "value": [ 19:34:52 DEBUG }, 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:34:52 DEBUG "pod": "ds-idrepo-0" 19:34:52 DEBUG "pod": "ds-cts-0" 19:34:52 DEBUG 19:34:52 DEBUG }, 19:34:52 DEBUG "result": [ 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "pod": "ds-cts-0" 19:34:52 DEBUG 19:34:52 DEBUG "metric": { 19:34:52 DEBUG }, 19:34:52 DEBUG 19:34:52 DEBUG }, 19:34:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 19:34:52 DEBUG "metric": { 19:34:52 DEBUG }, 19:34:52 DEBUG }, 19:34:52 DEBUG }, 19:34:52 DEBUG 19:34:52 DEBUG "value": [ 19:34:52 DEBUG { 19:34:52 DEBUG 19:34:52 DEBUG }, 19:34:52 DEBUG 19:34:52 DEBUG "instance": "10.142.0.107:9100" 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 19:34:52 DEBUG "value": [ 19:34:52 DEBUG }, 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "value": [ 19:34:52 DEBUG [...] 19:34:52 DEBUG 19:34:52 DEBUG "metric": { 19:34:52 DEBUG 19:34:52 DEBUG "value": [ 19:34:52 DEBUG [...] 19:34:52 DEBUG }, 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG }, 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:34:52 DEBUG }, 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 19:34:52 DEBUG [...] 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG [...] 19:34:52 DEBUG [...] 19:34:52 DEBUG 19:34:52 DEBUG [...] 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG [...] 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG [...] 19:34:52 DEBUG [...] 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG [...] 19:34:52 DEBUG 19:34:52 DEBUG [...] 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG [...] 19:34:52 DEBUG [...] 19:34:52 DEBUG [...] 19:34:52 DEBUG "pod": "ds-cts-2" 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG [...] 19:34:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG [...] 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG "instance": "10.142.0.43:9100" 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG }, 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG }, 19:34:52 DEBUG [...] 19:34:52 DEBUG 19:34:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG "instance": "10.142.0.43:9100" 19:34:52 DEBUG "pod": "ds-cts-2" 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG }, 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 19:34:52 DEBUG "instance": "10.142.0.43:9100" 19:34:52 DEBUG [...] 19:34:52 DEBUG 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG }, 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG }, 19:34:52 DEBUG }, 19:34:52 DEBUG "instance": "10.142.0.43:9100" 19:34:52 DEBUG [...] 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG }, 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG 19:34:52 DEBUG 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "pod": "ds-cts-0" 19:34:52 DEBUG 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG "pod": "ds-cts-0" 19:34:52 DEBUG "pod": "ds-idrepo-2" 19:34:52 DEBUG "pod": "ds-cts-2" 19:34:52 DEBUG "0" 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 19:34:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:52 DEBUG "pod": "ds-cts-2" 19:34:52 DEBUG "0.153" 19:34:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:34:52 DEBUG "instance": "10.142.0.43:9100" 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG }, 19:34:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG }, 19:34:52 DEBUG "30.03333333333333" 19:34:52 DEBUG 19:34:52 DEBUG "80748987.5" 19:34:52 DEBUG }, 19:34:52 DEBUG }, 19:34:52 DEBUG }, 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG 19:34:52 DEBUG }, 19:34:52 DEBUG }, 19:34:52 DEBUG ] 19:34:52 DEBUG }, 19:34:52 DEBUG }, 19:34:52 DEBUG "2663.9000000000005" 19:34:52 DEBUG "value": [ 19:34:52 DEBUG }, 19:34:52 DEBUG "80744791.46666667" 19:34:52 DEBUG "value": [ 19:34:52 DEBUG ] 19:34:52 DEBUG 19:34:52 DEBUG ] 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "value": [ 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG ] 19:34:52 DEBUG "0.10099999999998772" 19:34:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "value": [ 19:34:52 DEBUG } 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "value": [ 19:34:52 DEBUG ] 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG "value": [ 19:34:52 DEBUG ] 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG } 19:34:52 DEBUG "pod": "ds-cts-0" 19:34:52 DEBUG } 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG "0" 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG }, 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG "0.00013333333333333334" 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG "846331904" 19:34:52 DEBUG } 19:34:52 DEBUG "25593976.087309565" 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG } 19:34:52 DEBUG "0.27086666666670567" 19:34:52 DEBUG ] 19:34:52 DEBUG }, 19:34:52 DEBUG ] 19:34:52 DEBUG "70437.52019310673" 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG "value": [ 19:34:52 DEBUG "0" 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG "479801.5" 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG "value": [ 19:34:52 DEBUG } 19:34:52 DEBUG "0" 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG "3544226694144" 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG 1713638092, 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG "2662.5666666666666" 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG "11.995212903192506" 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG ] 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:34:52 DEBUG } 19:35:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638150" 19:35:50 ERROR [http_cmd]: ERROR 19:35:50 ERROR --- status code --- 19:35:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:35:50 ERROR --- http response --- 19:35:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:35:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:35:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638150" 19:35:50 INFO [http_cmd]: http status code OK 19:35:50 DEBUG --- status code --- 19:35:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:50 DEBUG --- http response --- 19:35:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:35:51 INFO 19:35:51 INFO [loop_until]: kubectl --namespace=xlou top pods 19:35:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:35:51 INFO [loop_until]: OK (rc = 0) 19:35:51 DEBUG --- stdout --- 19:35:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16969m 8650Mi am-767f6b67b5-qcqz2 16964m 9059Mi am-767f6b67b5-vbltz 18386m 8669Mi ds-cts-0 18515m 10243Mi ds-cts-1 17591m 10167Mi ds-cts-2 19046m 10221Mi ds-idrepo-0 7605m 3895Mi ds-idrepo-1 7335m 3908Mi ds-idrepo-2 7230m 3758Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 21m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8799m 717Mi 19:35:51 DEBUG --- stderr --- 19:35:51 DEBUG 19:35:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638152" 19:35:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638152" 19:35:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638152" 19:35:52 INFO [http_cmd]: http status code OK 19:35:52 DEBUG --- status code --- 19:35:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:52 DEBUG --- http response --- 19:35:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:52 DEBUG ----- output ----- 19:35:52 DEBUG { 19:35:52 DEBUG "status": "success", 19:35:52 DEBUG "data": { 19:35:52 DEBUG "resultType": "vector", 19:35:52 DEBUG "result": [ 19:35:52 DEBUG { 19:35:52 DEBUG "metric": { 19:35:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:52 DEBUG }, 19:35:52 DEBUG "value": [ 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG [...] 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:52 DEBUG }, 19:35:52 DEBUG "value": [ 19:35:52 DEBUG 1713638152, 19:35:52 DEBUG "2656.933333333334" 19:35:52 DEBUG ] 19:35:52 DEBUG } 19:35:52 DEBUG ] 19:35:52 DEBUG } 19:35:52 DEBUG } 19:35:52 INFO [http_cmd]: http status code OK 19:35:52 DEBUG --- status code --- 19:35:52 INFO [http_cmd]: http status code OK 19:35:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:52 DEBUG --- status code --- 19:35:52 DEBUG --- http response --- 19:35:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:52 DEBUG --- http response --- 19:35:52 DEBUG ----- output ----- 19:35:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:52 DEBUG { 19:35:52 DEBUG ----- output ----- 19:35:52 DEBUG "status": "success", 19:35:52 DEBUG { 19:35:52 DEBUG "data": { 19:35:52 DEBUG "resultType": "vector", 19:35:52 DEBUG "result": [ 19:35:52 DEBUG { 19:35:52 DEBUG "metric": { 19:35:52 DEBUG "pod": "ds-idrepo-0" 19:35:52 DEBUG }, 19:35:52 DEBUG "status": "success", 19:35:52 DEBUG "value": [ 19:35:52 DEBUG "data": { 19:35:52 DEBUG 19:35:52 DEBUG "resultType": "vector", 19:35:52 DEBUG 19:35:52 DEBUG "result": [ 19:35:52 DEBUG 19:35:52 DEBUG { 19:35:52 DEBUG [...] 19:35:52 DEBUG "metric": { 19:35:52 DEBUG 19:35:52 DEBUG "pod": "ds-idrepo-0" 19:35:52 DEBUG 19:35:52 DEBUG }, 19:35:52 DEBUG 19:35:52 DEBUG "value": [ 19:35:52 DEBUG "pod": "ds-idrepo-2" 19:35:52 DEBUG 19:35:52 DEBUG }, 19:35:52 DEBUG 19:35:52 DEBUG "value": [ 19:35:52 DEBUG 19:35:52 DEBUG 1713638152, 19:35:52 DEBUG [...] 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG "pod": "ds-idrepo-2" 19:35:52 DEBUG }, 19:35:52 DEBUG "value": [ 19:35:52 DEBUG 1713638152, 19:35:52 DEBUG "0" 19:35:52 DEBUG ] 19:35:52 DEBUG } 19:35:52 DEBUG ] 19:35:52 DEBUG } 19:35:52 DEBUG "0" 19:35:52 DEBUG } 19:35:52 DEBUG ] 19:35:52 DEBUG } 19:35:52 DEBUG ] 19:35:52 DEBUG } 19:35:52 DEBUG } 19:35:52 INFO 19:35:52 INFO [loop_until]: kubectl --namespace=xlou top node 19:35:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:35:52 INFO [loop_until]: OK (rc = 0) 19:35:52 DEBUG --- stdout --- 19:35:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6208Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7586m 25% 7637Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16838m 56% 11364Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 93m 0% 3345Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18469m 61% 13485Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 140m 0% 5224Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17269m 57% 11948Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14552m 48% 2958Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19529m 65% 10891Mi 9% gke-xlou-cdm-ds-0f601751-j488 8122m 27% 7170Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17522m 58% 15270Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8086m 27% 7876Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19173m 64% 12933Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9073m 57% 6448Mi 10% 19:35:52 DEBUG --- stderr --- 19:35:52 DEBUG 19:35:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638152" 19:35:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638152" 19:35:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638152" 19:35:52 INFO [http_cmd]: http status code OK 19:35:52 DEBUG --- status code --- 19:35:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:52 DEBUG --- http response --- 19:35:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:52 DEBUG ----- output ----- 19:35:52 DEBUG { 19:35:52 DEBUG "status": "success", 19:35:52 DEBUG "data": { 19:35:52 DEBUG "resultType": "vector", 19:35:52 DEBUG "result": [ 19:35:52 DEBUG { 19:35:52 DEBUG "metric": { 19:35:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:35:52 DEBUG }, 19:35:52 DEBUG "value": [ 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG [...] 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:35:52 DEBUG }, 19:35:52 INFO [http_cmd]: http status code OK 19:35:52 DEBUG "value": [ 19:35:52 DEBUG --- status code --- 19:35:52 DEBUG 1713638152, 19:35:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:52 DEBUG "8.891092753176997" 19:35:52 DEBUG ] 19:35:52 DEBUG } 19:35:52 DEBUG ] 19:35:52 DEBUG } 19:35:52 DEBUG } 19:35:52 DEBUG --- http response --- 19:35:52 INFO [http_cmd]: http status code OK 19:35:52 DEBUG --- status code --- 19:35:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:52 DEBUG --- http response --- 19:35:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:52 DEBUG ----- output ----- 19:35:52 DEBUG { 19:35:52 DEBUG "status": "success", 19:35:52 DEBUG "data": { 19:35:52 DEBUG "resultType": "vector", 19:35:52 DEBUG "result": [ 19:35:52 DEBUG { 19:35:52 DEBUG "metric": { 19:35:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:52 DEBUG }, 19:35:52 DEBUG "value": [ 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:35:52 DEBUG [...] 19:35:52 DEBUG ----- output ----- 19:35:52 DEBUG 19:35:52 DEBUG { 19:35:52 DEBUG 19:35:52 DEBUG "status": "success", 19:35:52 DEBUG 19:35:52 DEBUG "data": { 19:35:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:52 DEBUG "resultType": "vector", 19:35:52 DEBUG }, 19:35:52 DEBUG "value": [ 19:35:52 DEBUG 1713638152, 19:35:52 DEBUG "result": [ 19:35:52 DEBUG "2678.5333333333338" 19:35:52 DEBUG ] 19:35:52 DEBUG } 19:35:52 DEBUG ] 19:35:52 DEBUG } 19:35:52 DEBUG } 19:35:52 DEBUG { 19:35:52 DEBUG "metric": { 19:35:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:35:52 DEBUG }, 19:35:52 DEBUG "value": [ 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG [...] 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG 19:35:52 DEBUG "pod": "ds-cts-0" 19:35:52 DEBUG }, 19:35:52 DEBUG "value": [ 19:35:52 DEBUG 1713638152, 19:35:52 DEBUG "65156874.680441715" 19:35:52 DEBUG ] 19:35:52 DEBUG } 19:35:52 DEBUG ] 19:35:52 DEBUG } 19:35:52 DEBUG } 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 DEBUG --- status code --- 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638153" 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 DEBUG --- status code --- 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG --- status code --- 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 DEBUG --- status code --- 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG --- status code --- 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 DEBUG ----- output ----- 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG { 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG --- http response --- 19:35:53 INFO [http_cmd]: http status code OK 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:35:53 DEBUG { 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:53 DEBUG "data": { 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:35:53 DEBUG --- status code --- 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:35:53 DEBUG { 19:35:53 DEBUG { 19:35:53 DEBUG { 19:35:53 DEBUG "data": { 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:35:53 DEBUG { 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:53 DEBUG { 19:35:53 DEBUG { 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:35:53 DEBUG { 19:35:53 DEBUG { 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG { 19:35:53 DEBUG "result": [ 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG { 19:35:53 DEBUG --- http response --- 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG "data": { 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG "data": { 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG "data": { 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG { 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG { 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:35:53 DEBUG "data": { 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "result": [ 19:35:53 DEBUG "result": [ 19:35:53 DEBUG { 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "data": { 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG { 19:35:53 DEBUG "data": { 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "data": { 19:35:53 DEBUG "metric": { 19:35:53 DEBUG { 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG "data": { 19:35:53 DEBUG ----- output ----- 19:35:53 DEBUG "data": { 19:35:53 DEBUG { 19:35:53 DEBUG { 19:35:53 DEBUG { 19:35:53 DEBUG "data": { 19:35:53 DEBUG { 19:35:53 DEBUG "result": [ 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "result": [ 19:35:53 DEBUG "data": { 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "result": [ 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "instance": "10.142.0.107:9100" 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG "data": { 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG { 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "metric": { 19:35:53 DEBUG "metric": { 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG { 19:35:53 DEBUG "result": [ 19:35:53 DEBUG { 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "data": { 19:35:53 DEBUG "result": [ 19:35:53 DEBUG { 19:35:53 DEBUG "result": [ 19:35:53 DEBUG }, 19:35:53 DEBUG "data": { 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "result": [ 19:35:53 DEBUG "status": "success", 19:35:53 DEBUG "result": [ 19:35:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:53 DEBUG "instance": "10.142.0.107:9100" 19:35:53 DEBUG "data": { 19:35:53 DEBUG "result": [ 19:35:53 DEBUG "data": { 19:35:53 DEBUG "metric": { 19:35:53 DEBUG { 19:35:53 DEBUG "metric": { 19:35:53 DEBUG "result": [ 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG { 19:35:53 DEBUG "metric": { 19:35:53 DEBUG { 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "result": [ 19:35:53 DEBUG { 19:35:53 DEBUG "data": { 19:35:53 DEBUG { 19:35:53 DEBUG }, 19:35:53 DEBUG }, 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG { 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "pod": "ds-cts-0" 19:35:53 DEBUG "metric": { 19:35:53 DEBUG "instance": "10.142.0.107:9100" 19:35:53 DEBUG { 19:35:53 DEBUG "result": [ 19:35:53 DEBUG "metric": { 19:35:53 DEBUG "pod": "ds-cts-0" 19:35:53 DEBUG "metric": { 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG { 19:35:53 DEBUG "metric": { 19:35:53 DEBUG "resultType": "vector", 19:35:53 DEBUG "metric": { 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "result": [ 19:35:53 DEBUG "metric": { 19:35:53 DEBUG "result": [ 19:35:53 DEBUG }, 19:35:53 DEBUG "pod": "ds-idrepo-0" 19:35:53 DEBUG }, 19:35:53 DEBUG "metric": { 19:35:53 DEBUG { 19:35:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:35:53 DEBUG }, 19:35:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:53 DEBUG "result": [ 19:35:53 DEBUG 19:35:53 DEBUG "metric": { 19:35:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:35:53 DEBUG "result": [ 19:35:53 DEBUG "pod": "ds-idrepo-0" 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG { 19:35:53 DEBUG "pod": "ds-cts-0" 19:35:53 DEBUG { 19:35:53 DEBUG "value": [ 19:35:53 DEBUG }, 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:35:53 DEBUG }, 19:35:53 DEBUG }, 19:35:53 DEBUG "value": [ 19:35:53 DEBUG }, 19:35:53 DEBUG { 19:35:53 DEBUG [...] 19:35:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:53 DEBUG }, 19:35:53 DEBUG { 19:35:53 DEBUG }, 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "metric": { 19:35:53 DEBUG }, 19:35:53 DEBUG "metric": { 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "metric": { 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG }, 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "metric": { 19:35:53 DEBUG "value": [ 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "instance": "10.142.0.107:9100" 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "instance": "10.142.0.107:9100" 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG 19:35:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:35:53 DEBUG "metric": { 19:35:53 DEBUG 19:35:53 DEBUG [...] 19:35:53 DEBUG [...] 19:35:53 DEBUG }, 19:35:53 DEBUG 19:35:53 DEBUG }, 19:35:53 DEBUG [...] 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG [...] 19:35:53 DEBUG 19:35:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:53 DEBUG [...] 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG }, 19:35:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG }, 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG 19:35:53 DEBUG }, 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG [...] 19:35:53 DEBUG [...] 19:35:53 DEBUG [...] 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG 19:35:53 DEBUG "instance": "10.142.0.43:9100" 19:35:53 DEBUG }, 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG [...] 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG [...] 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "pod": "ds-cts-0" 19:35:53 DEBUG }, 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG [...] 19:35:53 DEBUG 19:35:53 DEBUG [...] 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:53 DEBUG "instance": "10.142.0.43:9100" 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "pod": "ds-cts-2" 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG [...] 19:35:53 DEBUG 19:35:53 DEBUG [...] 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG }, 19:35:53 DEBUG }, 19:35:53 DEBUG [...] 19:35:53 DEBUG 19:35:53 DEBUG [...] 19:35:53 DEBUG }, 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG 19:35:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:53 DEBUG 19:35:53 DEBUG [...] 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG [...] 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "value": [ 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "instance": "10.142.0.43:9100" 19:35:53 DEBUG "pod": "ds-idrepo-2" 19:35:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:35:53 DEBUG "0" 19:35:53 DEBUG 19:35:53 DEBUG "81066654.53333333" 19:35:53 DEBUG 19:35:53 DEBUG }, 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG 19:35:53 DEBUG "pod": "ds-cts-2" 19:35:53 DEBUG 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG "29.966666666666665" 19:35:53 DEBUG }, 19:35:53 DEBUG }, 19:35:53 DEBUG ] 19:35:53 DEBUG 19:35:53 DEBUG ] 19:35:53 DEBUG 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:53 DEBUG 19:35:53 DEBUG 19:35:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:53 DEBUG 19:35:53 DEBUG "0.153" 19:35:53 DEBUG "81063772.73333333" 19:35:53 DEBUG 19:35:53 DEBUG }, 19:35:53 DEBUG 19:35:53 DEBUG ] 19:35:53 DEBUG }, 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "value": [ 19:35:53 DEBUG } 19:35:53 DEBUG "pod": "ds-cts-2" 19:35:53 DEBUG } 19:35:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG }, 19:35:53 DEBUG "pod": "ds-cts-0" 19:35:53 DEBUG "pod": "ds-cts-2" 19:35:53 DEBUG }, 19:35:53 DEBUG 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG "instance": "10.142.0.43:9100" 19:35:53 DEBUG "value": [ 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG } 19:35:53 DEBUG "value": [ 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG ] 19:35:53 DEBUG }, 19:35:53 DEBUG ] 19:35:53 DEBUG }, 19:35:53 DEBUG "0" 19:35:53 DEBUG "value": [ 19:35:53 DEBUG }, 19:35:53 DEBUG }, 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:35:53 DEBUG ] 19:35:53 DEBUG ] 19:35:53 DEBUG }, 19:35:53 DEBUG "0.00046666666666666666" 19:35:53 DEBUG "instance": "10.142.0.43:9100" 19:35:53 DEBUG ] 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG "0" 19:35:53 DEBUG "847790080" 19:35:53 DEBUG } 19:35:53 DEBUG "value": [ 19:35:53 DEBUG } 19:35:53 DEBUG "value": [ 19:35:53 DEBUG ] 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG "value": [ 19:35:53 DEBUG "value": [ 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG }, 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG "value": [ 19:35:53 DEBUG ] 19:35:53 DEBUG }, 19:35:53 DEBUG } 19:35:53 DEBUG "0.08566666666668159" 19:35:53 DEBUG ] 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG } 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG } 19:35:53 DEBUG "480179.5" 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG "0" 19:35:53 DEBUG "value": [ 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG "3548535856128" 19:35:53 DEBUG "value": [ 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG "0" 19:35:53 DEBUG "2678.6666666666665" 19:35:53 DEBUG ] 19:35:53 DEBUG ] 19:35:53 DEBUG "0" 19:35:53 DEBUG "0" 19:35:53 DEBUG ] 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG 1713638153, 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG ] 19:35:53 DEBUG ] 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG "0.21833333333332897" 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG "2677.9333333333334" 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG ] 19:35:53 DEBUG ] 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG ] 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:35:53 DEBUG } 19:36:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638210" 19:36:50 ERROR [http_cmd]: ERROR 19:36:50 ERROR --- status code --- 19:36:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:36:50 ERROR --- http response --- 19:36:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:36:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:36:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638210" 19:36:50 INFO [http_cmd]: http status code OK 19:36:50 DEBUG --- status code --- 19:36:50 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:50 DEBUG --- http response --- 19:36:50 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:36:51 INFO 19:36:51 INFO [loop_until]: kubectl --namespace=xlou top pods 19:36:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:36:51 INFO [loop_until]: OK (rc = 0) 19:36:51 DEBUG --- stdout --- 19:36:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16642m 8651Mi am-767f6b67b5-qcqz2 17186m 9061Mi am-767f6b67b5-vbltz 17943m 8667Mi ds-cts-0 18860m 10216Mi ds-cts-1 17054m 10173Mi ds-cts-2 19433m 10219Mi ds-idrepo-0 7403m 3888Mi ds-idrepo-1 7244m 3908Mi ds-idrepo-2 7377m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8932m 719Mi 19:36:51 DEBUG --- stderr --- 19:36:51 DEBUG 19:36:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638212" 19:36:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638212" 19:36:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638212" 19:36:52 INFO [http_cmd]: http status code OK 19:36:52 DEBUG --- status code --- 19:36:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:52 DEBUG --- http response --- 19:36:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:52 DEBUG ----- output ----- 19:36:52 DEBUG { 19:36:52 DEBUG "status": "success", 19:36:52 DEBUG "data": { 19:36:52 DEBUG "resultType": "vector", 19:36:52 DEBUG "result": [ 19:36:52 DEBUG { 19:36:52 DEBUG "metric": { 19:36:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:52 DEBUG }, 19:36:52 DEBUG "value": [ 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG [...] 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:52 DEBUG }, 19:36:52 DEBUG "value": [ 19:36:52 DEBUG 1713638212, 19:36:52 DEBUG "2668.3666666666672" 19:36:52 DEBUG ] 19:36:52 DEBUG } 19:36:52 DEBUG ] 19:36:52 DEBUG } 19:36:52 DEBUG } 19:36:52 INFO [http_cmd]: http status code OK 19:36:52 INFO [http_cmd]: http status code OK 19:36:52 DEBUG --- status code --- 19:36:52 DEBUG --- status code --- 19:36:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:52 DEBUG --- http response --- 19:36:52 DEBUG --- http response --- 19:36:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:52 DEBUG ----- output ----- 19:36:52 DEBUG ----- output ----- 19:36:52 DEBUG { 19:36:52 DEBUG { 19:36:52 DEBUG "status": "success", 19:36:52 DEBUG "status": "success", 19:36:52 DEBUG "data": { 19:36:52 DEBUG "data": { 19:36:52 DEBUG "resultType": "vector", 19:36:52 DEBUG "resultType": "vector", 19:36:52 DEBUG "result": [ 19:36:52 DEBUG "result": [ 19:36:52 DEBUG { 19:36:52 DEBUG { 19:36:52 DEBUG "metric": { 19:36:52 DEBUG "metric": { 19:36:52 DEBUG "pod": "ds-idrepo-0" 19:36:52 DEBUG "pod": "ds-idrepo-0" 19:36:52 DEBUG }, 19:36:52 DEBUG }, 19:36:52 DEBUG "value": [ 19:36:52 DEBUG "value": [ 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG [...] 19:36:52 DEBUG [...] 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG "pod": "ds-idrepo-2" 19:36:52 DEBUG "pod": "ds-idrepo-2" 19:36:52 DEBUG }, 19:36:52 DEBUG }, 19:36:52 DEBUG "value": [ 19:36:52 DEBUG "value": [ 19:36:52 DEBUG 1713638212, 19:36:52 DEBUG 1713638212, 19:36:52 DEBUG "0" 19:36:52 DEBUG "0" 19:36:52 DEBUG ] 19:36:52 DEBUG ] 19:36:52 DEBUG } 19:36:52 DEBUG } 19:36:52 DEBUG ] 19:36:52 DEBUG } 19:36:52 DEBUG } 19:36:52 DEBUG ] 19:36:52 DEBUG } 19:36:52 DEBUG } 19:36:52 INFO 19:36:52 INFO [loop_until]: kubectl --namespace=xlou top node 19:36:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:36:52 INFO [loop_until]: OK (rc = 0) 19:36:52 DEBUG --- stdout --- 19:36:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6216Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7565m 25% 7626Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16808m 56% 11378Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3341Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18436m 61% 13481Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5193Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17226m 57% 11944Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14544m 48% 2957Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19409m 65% 10888Mi 9% gke-xlou-cdm-ds-0f601751-j488 8157m 27% 7171Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17017m 56% 15268Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8064m 27% 7871Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19335m 64% 12935Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9042m 56% 6445Mi 10% 19:36:52 DEBUG --- stderr --- 19:36:52 DEBUG 19:36:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638212" 19:36:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638212" 19:36:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638212" 19:36:52 INFO [http_cmd]: http status code OK 19:36:52 INFO [http_cmd]: http status code OK 19:36:52 DEBUG --- status code --- 19:36:52 DEBUG --- status code --- 19:36:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:52 DEBUG --- http response --- 19:36:52 DEBUG --- http response --- 19:36:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:52 DEBUG ----- output ----- 19:36:52 DEBUG { 19:36:52 DEBUG "status": "success", 19:36:52 DEBUG "data": { 19:36:52 DEBUG "resultType": "vector", 19:36:52 DEBUG "result": [ 19:36:52 DEBUG { 19:36:52 DEBUG "metric": { 19:36:52 DEBUG "pod": "ds-idrepo-1" 19:36:52 DEBUG }, 19:36:52 DEBUG "value": [ 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG [...] 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG ----- output ----- 19:36:52 DEBUG 19:36:52 INFO [http_cmd]: http status code OK 19:36:52 DEBUG { 19:36:52 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:36:52 DEBUG --- status code --- 19:36:52 DEBUG "status": "success", 19:36:52 DEBUG }, 19:36:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:52 DEBUG "data": { 19:36:52 DEBUG "resultType": "vector", 19:36:52 DEBUG --- http response --- 19:36:52 DEBUG "result": [ 19:36:52 DEBUG "value": [ 19:36:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:52 DEBUG { 19:36:52 DEBUG "metric": { 19:36:52 DEBUG ----- output ----- 19:36:52 DEBUG 1713638212, 19:36:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:36:52 DEBUG { 19:36:52 DEBUG "0" 19:36:52 DEBUG }, 19:36:52 DEBUG "value": [ 19:36:52 DEBUG 19:36:52 DEBUG "status": "success", 19:36:52 DEBUG ] 19:36:52 DEBUG 19:36:52 DEBUG 19:36:52 DEBUG } 19:36:52 DEBUG [...] 19:36:52 DEBUG ] 19:36:52 DEBUG "data": { 19:36:52 DEBUG 19:36:52 DEBUG } 19:36:52 DEBUG "resultType": "vector", 19:36:52 DEBUG 19:36:52 DEBUG } 19:36:52 DEBUG "result": [ 19:36:52 DEBUG 19:36:52 DEBUG { 19:36:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:36:52 DEBUG "metric": { 19:36:52 DEBUG }, 19:36:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:52 DEBUG "value": [ 19:36:52 DEBUG }, 19:36:52 DEBUG 1713638212, 19:36:52 DEBUG "value": [ 19:36:52 DEBUG "8.869782459798238" 19:36:52 DEBUG 19:36:52 DEBUG ] 19:36:52 DEBUG 19:36:52 DEBUG } 19:36:52 DEBUG 19:36:52 DEBUG [...] 19:36:52 DEBUG ] 19:36:52 DEBUG 19:36:52 DEBUG } 19:36:52 DEBUG 19:36:52 DEBUG } 19:36:52 DEBUG 19:36:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:52 DEBUG }, 19:36:52 DEBUG "value": [ 19:36:52 DEBUG 1713638212, 19:36:52 DEBUG "2671.733333333334" 19:36:52 DEBUG ] 19:36:52 DEBUG } 19:36:52 DEBUG ] 19:36:52 DEBUG } 19:36:52 DEBUG } 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638213" 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG { 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "data": { 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "result": [ 19:36:53 DEBUG { 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "instance": "10.142.0.107:9100" 19:36:53 DEBUG }, 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG "instance": "10.142.0.43:9100" 19:36:53 DEBUG }, 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG "3552755627008" 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638213" 19:36:53 DEBUG --- status code --- 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 DEBUG --- status code --- 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG --- status code --- 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG --- status code --- 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:53 INFO [http_cmd]: http status code OK 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:53 DEBUG { 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG --- status code --- 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG { 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:36:53 DEBUG { 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG { 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG { 19:36:53 DEBUG { 19:36:53 DEBUG { 19:36:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:36:53 DEBUG { 19:36:53 DEBUG { 19:36:53 DEBUG { 19:36:53 DEBUG { 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG { 19:36:53 DEBUG "data": { 19:36:53 DEBUG { 19:36:53 DEBUG { 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG { 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG --- http response --- 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "data": { 19:36:53 DEBUG { 19:36:53 DEBUG "data": { 19:36:53 DEBUG { 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "data": { 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "data": { 19:36:53 DEBUG "data": { 19:36:53 DEBUG "data": { 19:36:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:36:53 DEBUG "data": { 19:36:53 DEBUG "data": { 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "data": { 19:36:53 DEBUG "data": { 19:36:53 DEBUG "data": { 19:36:53 DEBUG "data": { 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "data": { 19:36:53 DEBUG "data": { 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "data": { 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG ----- output ----- 19:36:53 DEBUG { 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "data": { 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG { 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "status": "success", 19:36:53 DEBUG "result": [ 19:36:53 DEBUG { 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG { 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "result": [ 19:36:53 DEBUG { 19:36:53 DEBUG "result": [ 19:36:53 DEBUG { 19:36:53 DEBUG { 19:36:53 DEBUG { 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "data": { 19:36:53 DEBUG { 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "result": [ 19:36:53 DEBUG { 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "metric": { 19:36:53 DEBUG { 19:36:53 DEBUG { 19:36:53 DEBUG "pod": "ds-idrepo-1" 19:36:53 DEBUG { 19:36:53 DEBUG { 19:36:53 DEBUG "metric": { 19:36:53 DEBUG { 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "metric": { 19:36:53 DEBUG { 19:36:53 DEBUG "resultType": "vector", 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:36:53 DEBUG { 19:36:53 DEBUG "metric": { 19:36:53 DEBUG { 19:36:53 DEBUG "pod": "ds-idrepo-0" 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "metric": { 19:36:53 DEBUG }, 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "pod": "ds-cts-0" 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "instance": "10.142.0.107:9100" 19:36:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:53 DEBUG "pod": "ds-idrepo-1" 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "result": [ 19:36:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:53 DEBUG }, 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:53 DEBUG "metric": { 19:36:53 DEBUG }, 19:36:53 DEBUG "instance": "10.142.0.107:9100" 19:36:53 DEBUG "instance": "10.142.0.107:9100" 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:53 DEBUG "pod": "ds-idrepo-0" 19:36:53 DEBUG }, 19:36:53 DEBUG "pod": "ds-idrepo-1" 19:36:53 DEBUG }, 19:36:53 DEBUG }, 19:36:53 DEBUG }, 19:36:53 DEBUG "instance": "10.142.0.107:9100" 19:36:53 DEBUG { 19:36:53 DEBUG }, 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "pod": "ds-cts-0" 19:36:53 DEBUG }, 19:36:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:36:53 DEBUG "value": [ 19:36:53 DEBUG }, 19:36:53 DEBUG }, 19:36:53 DEBUG 19:36:53 DEBUG }, 19:36:53 DEBUG }, 19:36:53 DEBUG "value": [ 19:36:53 DEBUG }, 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "value": [ 19:36:53 DEBUG }, 19:36:53 DEBUG "metric": { 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 19:36:53 DEBUG }, 19:36:53 DEBUG "value": [ 19:36:53 DEBUG }, 19:36:53 DEBUG 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "pod": "ds-cts-0" 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG }, 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG [...] 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG [...] 19:36:53 DEBUG 19:36:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:36:53 DEBUG }, 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG "pod": "ds-idrepo-2" 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG }, 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG "pod": "ds-cts-2" 19:36:53 DEBUG 19:36:53 DEBUG "instance": "10.142.0.43:9100" 19:36:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:36:53 DEBUG }, 19:36:53 DEBUG 19:36:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG 19:36:53 DEBUG }, 19:36:53 DEBUG "instance": "10.142.0.43:9100" 19:36:53 DEBUG "instance": "10.142.0.43:9100" 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:53 DEBUG "pod": "ds-cts-2" 19:36:53 DEBUG }, 19:36:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:36:53 DEBUG }, 19:36:53 DEBUG }, 19:36:53 DEBUG 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 19:36:53 DEBUG }, 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:53 DEBUG "pod": "ds-cts-2" 19:36:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:36:53 DEBUG "value": [ 19:36:53 DEBUG }, 19:36:53 DEBUG }, 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG }, 19:36:53 DEBUG }, 19:36:53 DEBUG "value": [ 19:36:53 DEBUG }, 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "instance": "10.142.0.43:9100" 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "850350080" 19:36:53 DEBUG }, 19:36:53 DEBUG }, 19:36:53 DEBUG }, 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "49275.267454143854" 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG "value": [ 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG }, 19:36:53 DEBUG "0" 19:36:53 DEBUG "pod": "ds-cts-2" 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG ] 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "value": [ 19:36:53 DEBUG "0" 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG ] 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG "35.5" 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG "5.408900825899012" 19:36:53 DEBUG "80846218.7" 19:36:53 DEBUG "value": [ 19:36:53 DEBUG ] 19:36:53 DEBUG }, 19:36:53 DEBUG "0" 19:36:53 DEBUG } 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG ] 19:36:53 DEBUG "80851451.86666667" 19:36:53 DEBUG "0.19296666666672213" 19:36:53 DEBUG } 19:36:53 DEBUG "2671.1" 19:36:53 DEBUG "0" 19:36:53 DEBUG ] 19:36:53 DEBUG "0.1535" 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG } 19:36:53 DEBUG "value": [ 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG "480356.5" 19:36:53 DEBUG "0.0008333333333333333" 19:36:53 DEBUG "2671.9666666666667" 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG "0.07100000000001312" 19:36:53 DEBUG ] 19:36:53 DEBUG 1713638213, 19:36:53 DEBUG "0" 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG ] 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:36:53 DEBUG } 19:37:50 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638270" 19:37:50 ERROR [http_cmd]: ERROR 19:37:50 ERROR --- status code --- 19:37:50 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:37:50 ERROR --- http response --- 19:37:50 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:37:50 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:37:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638271" 19:37:51 INFO [http_cmd]: http status code OK 19:37:51 DEBUG --- status code --- 19:37:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:51 DEBUG --- http response --- 19:37:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:37:51 INFO 19:37:51 INFO [loop_until]: kubectl --namespace=xlou top pods 19:37:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:37:52 INFO [loop_until]: OK (rc = 0) 19:37:52 DEBUG --- stdout --- 19:37:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16821m 8655Mi am-767f6b67b5-qcqz2 17319m 9061Mi am-767f6b67b5-vbltz 18393m 8667Mi ds-cts-0 19058m 10248Mi ds-cts-1 17524m 10167Mi ds-cts-2 19309m 10225Mi ds-idrepo-0 7396m 3888Mi ds-idrepo-1 7527m 3922Mi ds-idrepo-2 7281m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 13m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8741m 721Mi 19:37:52 DEBUG --- stderr --- 19:37:52 DEBUG 19:37:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638272" 19:37:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638272" 19:37:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638272" 19:37:52 INFO [http_cmd]: http status code OK 19:37:52 DEBUG --- status code --- 19:37:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:52 DEBUG --- http response --- 19:37:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:52 DEBUG ----- output ----- 19:37:52 DEBUG { 19:37:52 DEBUG "status": "success", 19:37:52 DEBUG "data": { 19:37:52 DEBUG "resultType": "vector", 19:37:52 INFO [http_cmd]: http status code OK 19:37:52 DEBUG "result": [ 19:37:52 DEBUG --- status code --- 19:37:52 DEBUG { 19:37:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:52 DEBUG "metric": { 19:37:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:52 DEBUG --- http response --- 19:37:52 INFO [http_cmd]: http status code OK 19:37:52 DEBUG }, 19:37:52 DEBUG --- status code --- 19:37:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:52 DEBUG "value": [ 19:37:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:52 DEBUG ----- output ----- 19:37:52 DEBUG 19:37:52 DEBUG --- http response --- 19:37:52 DEBUG { 19:37:52 DEBUG 19:37:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:52 DEBUG "status": "success", 19:37:52 DEBUG 19:37:52 DEBUG ----- output ----- 19:37:52 DEBUG "data": { 19:37:52 DEBUG [...] 19:37:52 DEBUG { 19:37:52 DEBUG "resultType": "vector", 19:37:52 DEBUG 19:37:52 DEBUG "status": "success", 19:37:52 DEBUG "result": [ 19:37:52 DEBUG 19:37:52 DEBUG "data": { 19:37:52 DEBUG { 19:37:52 DEBUG 19:37:52 DEBUG "resultType": "vector", 19:37:52 DEBUG "metric": { 19:37:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:52 DEBUG "result": [ 19:37:52 DEBUG "pod": "ds-idrepo-0" 19:37:52 DEBUG }, 19:37:52 DEBUG { 19:37:52 DEBUG }, 19:37:52 DEBUG "value": [ 19:37:52 DEBUG "metric": { 19:37:52 DEBUG "value": [ 19:37:52 DEBUG 1713638272, 19:37:52 DEBUG "pod": "ds-idrepo-0" 19:37:52 DEBUG }, 19:37:52 DEBUG "value": [ 19:37:52 DEBUG 19:37:52 DEBUG 19:37:52 DEBUG 19:37:52 DEBUG [...] 19:37:52 DEBUG "2664.0333333333338" 19:37:52 DEBUG 19:37:52 DEBUG 19:37:52 DEBUG ] 19:37:52 DEBUG 19:37:52 DEBUG 19:37:52 DEBUG } 19:37:52 DEBUG "pod": "ds-idrepo-2" 19:37:52 DEBUG 19:37:52 DEBUG ] 19:37:52 DEBUG }, 19:37:52 DEBUG 19:37:52 DEBUG } 19:37:52 DEBUG "value": [ 19:37:52 DEBUG [...] 19:37:52 DEBUG } 19:37:52 DEBUG 1713638272, 19:37:52 DEBUG 19:37:52 DEBUG "0" 19:37:52 DEBUG 19:37:52 DEBUG ] 19:37:52 DEBUG 19:37:52 DEBUG } 19:37:52 DEBUG "pod": "ds-idrepo-2" 19:37:52 DEBUG ] 19:37:52 DEBUG }, 19:37:52 DEBUG } 19:37:52 DEBUG "value": [ 19:37:52 DEBUG } 19:37:52 DEBUG 1713638272, 19:37:52 DEBUG "0" 19:37:52 DEBUG ] 19:37:52 DEBUG } 19:37:52 DEBUG ] 19:37:52 DEBUG } 19:37:52 DEBUG } 19:37:52 INFO 19:37:52 INFO [loop_until]: kubectl --namespace=xlou top node 19:37:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:37:52 INFO [loop_until]: OK (rc = 0) 19:37:52 DEBUG --- stdout --- 19:37:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6216Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7550m 25% 7636Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17036m 57% 11371Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 96m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18928m 63% 13486Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5191Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17284m 57% 11948Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14548m 48% 2960Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18767m 62% 10890Mi 9% gke-xlou-cdm-ds-0f601751-j488 8125m 27% 7174Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17051m 57% 15268Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8126m 27% 7877Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18953m 63% 12919Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9072m 57% 6447Mi 10% 19:37:52 DEBUG --- stderr --- 19:37:52 DEBUG 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638273" 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG { 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 DEBUG { 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "data": { 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG "data": { 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "result": [ 19:37:53 DEBUG { 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:53 DEBUG "result": [ 19:37:53 DEBUG { 19:37:53 DEBUG }, 19:37:53 DEBUG "metric": { 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:37:53 DEBUG { 19:37:53 DEBUG 19:37:53 DEBUG }, 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "data": { 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG "result": [ 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG { 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG "metric": { 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:37:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:53 DEBUG 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:53 DEBUG 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG }, 19:37:53 DEBUG 19:37:53 DEBUG "2666.8333333333335" 19:37:53 DEBUG "value": [ 19:37:53 DEBUG 19:37:53 DEBUG ] 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG [...] 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG "517.5801611119886" 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG 19:37:53 DEBUG } 19:37:53 DEBUG 19:37:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:37:53 DEBUG } 19:37:53 DEBUG }, 19:37:53 DEBUG "value": [ 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG "8.869071584106015" 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638273" 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG { 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "data": { 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "result": [ 19:37:53 DEBUG { 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "instance": "10.142.0.107:9100" 19:37:53 DEBUG }, 19:37:53 DEBUG "value": [ 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG "instance": "10.142.0.43:9100" 19:37:53 DEBUG }, 19:37:53 DEBUG "value": [ 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG "3557158603776" 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638273" 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 DEBUG --- status code --- 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- status code --- 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG --- http response --- 19:37:53 INFO [http_cmd]: http status code OK 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG --- status code --- 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG { 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG { 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG { 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG { 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG --- http response --- 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG { 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG { 19:37:53 DEBUG "data": { 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:37:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG "data": { 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG "data": { 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG ----- output ----- 19:37:53 DEBUG { 19:37:53 DEBUG "data": { 19:37:53 DEBUG "data": { 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "data": { 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "data": { 19:37:53 DEBUG { 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "data": { 19:37:53 DEBUG "data": { 19:37:53 DEBUG "data": { 19:37:53 DEBUG "data": { 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG { 19:37:53 DEBUG "data": { 19:37:53 DEBUG "data": { 19:37:53 DEBUG "data": { 19:37:53 DEBUG { 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "status": "success", 19:37:53 DEBUG "data": { 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "data": { 19:37:53 DEBUG "data": { 19:37:53 DEBUG { 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "data": { 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "result": [ 19:37:53 DEBUG { 19:37:53 DEBUG "instance": "10.142.0.107:9100" 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "pod": "ds-cts-0" 19:37:53 DEBUG { 19:37:53 DEBUG "resultType": "vector", 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "result": [ 19:37:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG "metric": { 19:37:53 DEBUG }, 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG }, 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "result": [ 19:37:53 DEBUG { 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "instance": "10.142.0.107:9100" 19:37:53 DEBUG { 19:37:53 DEBUG { 19:37:53 DEBUG }, 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "instance": "10.142.0.107:9100" 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:53 DEBUG { 19:37:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:53 DEBUG "pod": "ds-cts-0" 19:37:53 DEBUG }, 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "metric": { 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:53 DEBUG "pod": "ds-idrepo-0" 19:37:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:37:53 DEBUG "pod": "ds-idrepo-0" 19:37:53 DEBUG }, 19:37:53 DEBUG 19:37:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:37:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:37:53 DEBUG "instance": "10.142.0.107:9100" 19:37:53 DEBUG 19:37:53 DEBUG }, 19:37:53 DEBUG "metric": { 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:37:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:37:53 DEBUG 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG "value": [ 19:37:53 DEBUG 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "pod": "ds-cts-0" 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG }, 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG "value": [ 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG [...] 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG "instance": "10.142.0.43:9100" 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG "pod": "ds-cts-2" 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG [...] 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG }, 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG }, 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG "pod": "ds-cts-2" 19:37:53 DEBUG "instance": "10.142.0.43:9100" 19:37:53 DEBUG 19:37:53 DEBUG }, 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:53 DEBUG "instance": "10.142.0.43:9100" 19:37:53 DEBUG "value": [ 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:53 DEBUG "pod": "ds-idrepo-2" 19:37:53 DEBUG 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:37:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:53 DEBUG "instance": "10.142.0.43:9100" 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG }, 19:37:53 DEBUG "pod": "ds-cts-2" 19:37:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:37:53 DEBUG 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG "pod": "ds-cts-2" 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG "29.966666666666665" 19:37:53 DEBUG "value": [ 19:37:53 DEBUG }, 19:37:53 DEBUG }, 19:37:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG }, 19:37:53 DEBUG "2665.8" 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG }, 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG "80952906.33333333" 19:37:53 DEBUG "0.0870000000000156" 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG ] 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "value": [ 19:37:53 DEBUG }, 19:37:53 DEBUG "0" 19:37:53 DEBUG "80948081.33333334" 19:37:53 DEBUG "value": [ 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG "value": [ 19:37:53 DEBUG "0.154" 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG } 19:37:53 DEBUG "0" 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG "value": [ 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG "0" 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG "0" 19:37:53 DEBUG "0" 19:37:53 DEBUG "0.2173666666667136" 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG "0.0011666666666666665" 19:37:53 DEBUG "480474.5" 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG ] 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG "0" 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG "0" 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG "853049344" 19:37:53 DEBUG } 19:37:53 DEBUG 1713638273, 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG "2666.766666666667" 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG ] 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:37:53 DEBUG } 19:38:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638331" 19:38:51 ERROR [http_cmd]: ERROR 19:38:51 ERROR --- status code --- 19:38:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:38:51 ERROR --- http response --- 19:38:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:38:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:38:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638331" 19:38:51 INFO [http_cmd]: http status code OK 19:38:51 DEBUG --- status code --- 19:38:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:51 DEBUG --- http response --- 19:38:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:38:52 INFO 19:38:52 INFO [loop_until]: kubectl --namespace=xlou top pods 19:38:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:38:52 INFO [loop_until]: OK (rc = 0) 19:38:52 DEBUG --- stdout --- 19:38:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17111m 8651Mi am-767f6b67b5-qcqz2 16852m 9059Mi am-767f6b67b5-vbltz 17779m 8666Mi ds-cts-0 18153m 10234Mi ds-cts-1 16779m 10191Mi ds-cts-2 19142m 10224Mi ds-idrepo-0 7357m 3887Mi ds-idrepo-1 7439m 3905Mi ds-idrepo-2 7376m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8939m 723Mi 19:38:52 DEBUG --- stderr --- 19:38:52 DEBUG 19:38:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638332" 19:38:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638332" 19:38:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638332" 19:38:52 INFO [http_cmd]: http status code OK 19:38:52 INFO [http_cmd]: http status code OK 19:38:52 INFO [http_cmd]: http status code OK 19:38:52 DEBUG --- status code --- 19:38:52 DEBUG --- status code --- 19:38:52 DEBUG --- status code --- 19:38:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:52 DEBUG --- http response --- 19:38:52 DEBUG --- http response --- 19:38:52 DEBUG --- http response --- 19:38:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:52 DEBUG ----- output ----- 19:38:52 DEBUG ----- output ----- 19:38:52 DEBUG ----- output ----- 19:38:52 DEBUG { 19:38:52 DEBUG { 19:38:52 DEBUG { 19:38:52 DEBUG "status": "success", 19:38:52 DEBUG "status": "success", 19:38:52 DEBUG "status": "success", 19:38:52 DEBUG "data": { 19:38:52 DEBUG "data": { 19:38:52 DEBUG "data": { 19:38:52 DEBUG "resultType": "vector", 19:38:52 DEBUG "resultType": "vector", 19:38:52 DEBUG "resultType": "vector", 19:38:52 DEBUG "result": [ 19:38:52 DEBUG "result": [ 19:38:52 DEBUG "result": [ 19:38:52 DEBUG { 19:38:52 DEBUG { 19:38:52 DEBUG { 19:38:52 DEBUG "metric": { 19:38:52 DEBUG "metric": { 19:38:52 DEBUG "metric": { 19:38:52 DEBUG "pod": "ds-idrepo-0" 19:38:52 DEBUG "pod": "ds-idrepo-0" 19:38:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:52 DEBUG }, 19:38:52 DEBUG }, 19:38:52 DEBUG }, 19:38:52 DEBUG "value": [ 19:38:52 DEBUG "value": [ 19:38:52 DEBUG "value": [ 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG [...] 19:38:52 DEBUG [...] 19:38:52 DEBUG [...] 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG 19:38:52 DEBUG "pod": "ds-idrepo-2" 19:38:52 DEBUG "pod": "ds-idrepo-2" 19:38:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:52 DEBUG }, 19:38:52 DEBUG }, 19:38:52 DEBUG }, 19:38:52 DEBUG "value": [ 19:38:52 DEBUG 1713638332, 19:38:52 DEBUG "value": [ 19:38:52 DEBUG "value": [ 19:38:52 DEBUG "0" 19:38:52 DEBUG 1713638332, 19:38:52 DEBUG 1713638332, 19:38:52 DEBUG ] 19:38:52 DEBUG "2679.6000000000004" 19:38:52 DEBUG "0" 19:38:52 DEBUG } 19:38:52 DEBUG ] 19:38:52 DEBUG ] 19:38:52 DEBUG ] 19:38:52 DEBUG } 19:38:52 DEBUG } 19:38:52 DEBUG } 19:38:52 DEBUG ] 19:38:52 DEBUG ] 19:38:52 DEBUG } 19:38:52 DEBUG } 19:38:52 DEBUG } 19:38:52 DEBUG } 19:38:52 DEBUG } 19:38:52 INFO 19:38:52 INFO [loop_until]: kubectl --namespace=xlou top node 19:38:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:38:53 INFO [loop_until]: OK (rc = 0) 19:38:53 DEBUG --- stdout --- 19:38:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 60m 0% 6223Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7454m 24% 7632Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16862m 56% 11373Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3364Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18844m 63% 13475Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 141m 0% 5216Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17171m 57% 11947Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14483m 48% 2957Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19369m 64% 10896Mi 9% gke-xlou-cdm-ds-0f601751-j488 8162m 27% 7180Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16780m 56% 15263Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8086m 27% 7861Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19217m 64% 12942Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8726m 54% 6446Mi 10% 19:38:53 DEBUG --- stderr --- 19:38:53 DEBUG 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638333" 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG --- http response --- 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG { 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG "data": { 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG { 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG { 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "data": { 19:38:53 DEBUG { 19:38:53 DEBUG "data": { 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "result": [ 19:38:53 DEBUG { 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:53 DEBUG "metric": { 19:38:53 DEBUG { 19:38:53 DEBUG }, 19:38:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "value": [ 19:38:53 DEBUG }, 19:38:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG }, 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG }, 19:38:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG }, 19:38:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG }, 19:38:53 DEBUG "2664.5666666666666" 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG ] 19:38:53 DEBUG "0" 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG ] 19:38:53 DEBUG ] 19:38:53 DEBUG "8.80378695886431" 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638333" 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG { 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "data": { 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "result": [ 19:38:53 DEBUG { 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "instance": "10.142.0.107:9100" 19:38:53 DEBUG }, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG "instance": "10.142.0.43:9100" 19:38:53 DEBUG }, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG "3561169249280" 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638333" 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG --- status code --- 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG --- status code --- 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG --- status code --- 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG --- status code --- 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG --- http response --- 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:53 INFO [http_cmd]: http status code OK 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG { 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG --- status code --- 19:38:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG { 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG { 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:38:53 DEBUG "data": { 19:38:53 DEBUG { 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG { 19:38:53 DEBUG --- http response --- 19:38:53 DEBUG { 19:38:53 DEBUG { 19:38:53 DEBUG { 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG "data": { 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG { 19:38:53 DEBUG { 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG { 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "result": [ 19:38:53 DEBUG { 19:38:53 DEBUG { 19:38:53 DEBUG "data": { 19:38:53 DEBUG { 19:38:53 DEBUG { 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "data": { 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "data": { 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG "data": { 19:38:53 DEBUG "data": { 19:38:53 DEBUG "data": { 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "data": { 19:38:53 DEBUG ----- output ----- 19:38:53 DEBUG { 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG { 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "data": { 19:38:53 DEBUG "data": { 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG { 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG { 19:38:53 DEBUG { 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "data": { 19:38:53 DEBUG "data": { 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "data": { 19:38:53 DEBUG "data": { 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "data": { 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "status": "success", 19:38:53 DEBUG "instance": "10.142.0.107:9100" 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG { 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "result": [ 19:38:53 DEBUG { 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "result": [ 19:38:53 DEBUG { 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "result": [ 19:38:53 DEBUG { 19:38:53 DEBUG { 19:38:53 DEBUG { 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "pod": "ds-idrepo-0" 19:38:53 DEBUG }, 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "result": [ 19:38:53 DEBUG { 19:38:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:38:53 DEBUG "metric": { 19:38:53 DEBUG { 19:38:53 DEBUG "pod": "ds-cts-0" 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG { 19:38:53 DEBUG "metric": { 19:38:53 DEBUG { 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "data": { 19:38:53 DEBUG }, 19:38:53 DEBUG "data": { 19:38:53 DEBUG "value": [ 19:38:53 DEBUG { 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:38:53 DEBUG { 19:38:53 DEBUG "metric": { 19:38:53 DEBUG }, 19:38:53 DEBUG "pod": "ds-cts-0" 19:38:53 DEBUG "metric": { 19:38:53 DEBUG }, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG "value": [ 19:38:53 DEBUG "resultType": "vector", 19:38:53 DEBUG 19:38:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:53 DEBUG { 19:38:53 DEBUG }, 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:53 DEBUG "value": [ 19:38:53 DEBUG }, 19:38:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:53 DEBUG 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "instance": "10.142.0.107:9100" 19:38:53 DEBUG }, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG "result": [ 19:38:53 DEBUG "result": [ 19:38:53 DEBUG 19:38:53 DEBUG }, 19:38:53 DEBUG "metric": { 19:38:53 DEBUG "value": [ 19:38:53 DEBUG "instance": "10.142.0.107:9100" 19:38:53 DEBUG }, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG "value": [ 19:38:53 DEBUG }, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG { 19:38:53 DEBUG "metric": { 19:38:53 DEBUG }, 19:38:53 DEBUG 19:38:53 DEBUG "pod": "ds-idrepo-0" 19:38:53 DEBUG 19:38:53 DEBUG { 19:38:53 DEBUG "metric": { 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG "instance": "10.142.0.107:9100" 19:38:53 DEBUG 19:38:53 DEBUG }, 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG "pod": "ds-cts-0" 19:38:53 DEBUG }, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG [...] 19:38:53 DEBUG }, 19:38:53 DEBUG 19:38:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:38:53 DEBUG { 19:38:53 DEBUG "metric": { 19:38:53 DEBUG 19:38:53 DEBUG }, 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG }, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 19:38:53 DEBUG }, 19:38:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:38:53 DEBUG }, 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:53 DEBUG }, 19:38:53 DEBUG 19:38:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:38:53 DEBUG [...] 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG [...] 19:38:53 DEBUG [...] 19:38:53 DEBUG }, 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG }, 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG }, 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG "pod": "ds-cts-2" 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG "pod": "ds-cts-2" 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG }, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:38:53 DEBUG "instance": "10.142.0.43:9100" 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG "pod": "ds-cts-2" 19:38:53 DEBUG }, 19:38:53 DEBUG }, 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG }, 19:38:53 DEBUG "2664.1666666666665" 19:38:53 DEBUG [...] 19:38:53 DEBUG 19:38:53 DEBUG "pod": "ds-idrepo-2" 19:38:53 DEBUG 19:38:53 DEBUG "0" 19:38:53 DEBUG [...] 19:38:53 DEBUG }, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG }, 19:38:53 DEBUG "0" 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG "instance": "10.142.0.43:9100" 19:38:53 DEBUG "pod": "ds-cts-2" 19:38:53 DEBUG "58046.95679644027" 19:38:53 DEBUG "value": [ 19:38:53 DEBUG ] 19:38:53 DEBUG 19:38:53 DEBUG }, 19:38:53 DEBUG "854564864" 19:38:53 DEBUG 19:38:53 DEBUG ] 19:38:53 DEBUG 19:38:53 DEBUG 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG ] 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG }, 19:38:53 DEBUG "value": [ 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG "0.00030000000000000003" 19:38:53 DEBUG 19:38:53 DEBUG "value": [ 19:38:53 DEBUG ] 19:38:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:53 DEBUG }, 19:38:53 DEBUG 19:38:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:53 DEBUG "13.671565773518351" 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG 19:38:53 DEBUG "479936" 19:38:53 DEBUG "0.17439999999999808" 19:38:53 DEBUG }, 19:38:53 DEBUG ] 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG 19:38:53 DEBUG "instance": "10.142.0.43:9100" 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG "value": [ 19:38:53 DEBUG ] 19:38:53 DEBUG "0.07433333333332257" 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG "0" 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG "0" 19:38:53 DEBUG 19:38:53 DEBUG }, 19:38:53 DEBUG } 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG "value": [ 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG }, 19:38:53 DEBUG ] 19:38:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:38:53 DEBUG "value": [ 19:38:53 DEBUG ] 19:38:53 DEBUG "0.154" 19:38:53 DEBUG ] 19:38:53 DEBUG "instance": "10.142.0.43:9100" 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG "value": [ 19:38:53 DEBUG } 19:38:53 DEBUG }, 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG }, 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG "21.3" 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG "80854598.16666666" 19:38:53 DEBUG "value": [ 19:38:53 DEBUG "0" 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG "value": [ 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG ] 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG "2664.233333333333" 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG 1713638333, 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG "80849768.53333333" 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG ] 19:38:53 DEBUG } 19:38:53 DEBUG } 19:38:53 DEBUG } 19:39:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638391" 19:39:51 ERROR [http_cmd]: ERROR 19:39:51 ERROR --- status code --- 19:39:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:39:51 ERROR --- http response --- 19:39:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:39:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:39:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638391" 19:39:51 INFO [http_cmd]: http status code OK 19:39:51 DEBUG --- status code --- 19:39:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:51 DEBUG --- http response --- 19:39:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:39:52 INFO 19:39:52 INFO [loop_until]: kubectl --namespace=xlou top pods 19:39:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:39:52 INFO [loop_until]: OK (rc = 0) 19:39:52 DEBUG --- stdout --- 19:39:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16789m 8653Mi am-767f6b67b5-qcqz2 17068m 9059Mi am-767f6b67b5-vbltz 18415m 8668Mi ds-cts-0 18878m 10220Mi ds-cts-1 17382m 10167Mi ds-cts-2 19523m 10235Mi ds-idrepo-0 7066m 3888Mi ds-idrepo-1 7210m 3905Mi ds-idrepo-2 7492m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 16m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8747m 725Mi 19:39:52 DEBUG --- stderr --- 19:39:52 DEBUG 19:39:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638392" 19:39:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638392" 19:39:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638392" 19:39:52 INFO [http_cmd]: http status code OK 19:39:52 DEBUG --- status code --- 19:39:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:52 DEBUG --- http response --- 19:39:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:52 DEBUG ----- output ----- 19:39:52 DEBUG { 19:39:52 DEBUG "status": "success", 19:39:52 DEBUG "data": { 19:39:52 DEBUG "resultType": "vector", 19:39:52 DEBUG "result": [ 19:39:52 DEBUG { 19:39:52 DEBUG "metric": { 19:39:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:52 DEBUG }, 19:39:52 DEBUG "value": [ 19:39:52 DEBUG 19:39:52 DEBUG 19:39:52 DEBUG 19:39:52 DEBUG [...] 19:39:52 DEBUG 19:39:52 DEBUG 19:39:52 DEBUG 19:39:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:52 DEBUG }, 19:39:52 DEBUG "value": [ 19:39:52 DEBUG 1713638392, 19:39:52 DEBUG "2670.3000000000006" 19:39:52 DEBUG ] 19:39:52 DEBUG } 19:39:52 DEBUG ] 19:39:52 DEBUG } 19:39:52 DEBUG } 19:39:52 INFO [http_cmd]: http status code OK 19:39:52 DEBUG --- status code --- 19:39:52 INFO [http_cmd]: http status code OK 19:39:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:52 DEBUG --- status code --- 19:39:52 DEBUG --- http response --- 19:39:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:52 DEBUG --- http response --- 19:39:52 DEBUG ----- output ----- 19:39:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:52 DEBUG { 19:39:52 DEBUG ----- output ----- 19:39:52 DEBUG "status": "success", 19:39:52 DEBUG { 19:39:52 DEBUG "data": { 19:39:52 DEBUG "status": "success", 19:39:52 DEBUG "resultType": "vector", 19:39:52 DEBUG "data": { 19:39:52 DEBUG "result": [ 19:39:52 DEBUG "resultType": "vector", 19:39:52 DEBUG { 19:39:52 DEBUG "result": [ 19:39:52 DEBUG "metric": { 19:39:52 DEBUG { 19:39:52 DEBUG "pod": "ds-idrepo-0" 19:39:52 DEBUG "metric": { 19:39:52 DEBUG }, 19:39:52 DEBUG "pod": "ds-idrepo-0" 19:39:52 DEBUG "value": [ 19:39:52 DEBUG }, 19:39:52 DEBUG 19:39:52 DEBUG "value": [ 19:39:52 DEBUG 19:39:52 DEBUG 19:39:52 DEBUG 19:39:52 DEBUG 19:39:52 DEBUG [...] 19:39:52 DEBUG 19:39:52 DEBUG 19:39:52 DEBUG [...] 19:39:52 DEBUG 19:39:52 DEBUG 19:39:52 DEBUG 19:39:52 DEBUG 19:39:52 DEBUG "pod": "ds-idrepo-2" 19:39:52 DEBUG 19:39:52 DEBUG }, 19:39:52 DEBUG "pod": "ds-idrepo-2" 19:39:52 DEBUG "value": [ 19:39:52 DEBUG 1713638392, 19:39:52 DEBUG "0" 19:39:52 DEBUG ] 19:39:52 DEBUG } 19:39:52 DEBUG ] 19:39:52 DEBUG } 19:39:52 DEBUG } 19:39:52 DEBUG }, 19:39:52 DEBUG "value": [ 19:39:52 DEBUG 1713638392, 19:39:52 DEBUG "0" 19:39:52 DEBUG ] 19:39:52 DEBUG } 19:39:52 DEBUG ] 19:39:52 DEBUG } 19:39:52 DEBUG } 19:39:53 INFO 19:39:53 INFO [loop_until]: kubectl --namespace=xlou top node 19:39:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:39:53 INFO [loop_until]: OK (rc = 0) 19:39:53 DEBUG --- stdout --- 19:39:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6214Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7542m 25% 7636Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16951m 56% 11368Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 96m 0% 3353Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18536m 62% 13479Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5199Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17221m 57% 11939Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14507m 48% 2960Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19450m 65% 10886Mi 9% gke-xlou-cdm-ds-0f601751-j488 8116m 27% 7185Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17057m 57% 15271Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8086m 27% 7868Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19101m 63% 12929Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9051m 56% 6447Mi 10% 19:39:53 DEBUG --- stderr --- 19:39:53 DEBUG 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG { 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "data": { 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG "result": [ 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG { 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG "metric": { 19:39:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:53 DEBUG }, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:53 DEBUG 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG { 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 DEBUG 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:53 DEBUG "data": { 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG }, 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "result": [ 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:39:53 DEBUG { 19:39:53 DEBUG "2668.4666666666667" 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG "metric": { 19:39:53 DEBUG ] 19:39:53 DEBUG { 19:39:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:39:53 DEBUG } 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG }, 19:39:53 DEBUG ] 19:39:53 DEBUG "data": { 19:39:53 DEBUG "value": [ 19:39:53 DEBUG } 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG 19:39:53 DEBUG } 19:39:53 DEBUG "result": [ 19:39:53 DEBUG 19:39:53 DEBUG { 19:39:53 DEBUG 19:39:53 DEBUG "metric": { 19:39:53 DEBUG [...] 19:39:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:39:53 DEBUG 19:39:53 DEBUG }, 19:39:53 DEBUG 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:39:53 DEBUG 19:39:53 DEBUG }, 19:39:53 DEBUG 19:39:53 DEBUG "value": [ 19:39:53 DEBUG [...] 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG 19:39:53 DEBUG "8.914937729400167" 19:39:53 DEBUG 19:39:53 DEBUG ] 19:39:53 DEBUG 19:39:53 DEBUG } 19:39:53 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:39:53 DEBUG ] 19:39:53 DEBUG }, 19:39:53 DEBUG } 19:39:53 DEBUG "value": [ 19:39:53 DEBUG } 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG "1120.6442149729864" 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638393" 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG { 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "data": { 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "result": [ 19:39:53 DEBUG { 19:39:53 DEBUG "metric": { 19:39:53 DEBUG "instance": "10.142.0.107:9100" 19:39:53 DEBUG }, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG "instance": "10.142.0.43:9100" 19:39:53 DEBUG }, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG "3565479973888" 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638393" 19:39:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638393" 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 DEBUG --- status code --- 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 DEBUG --- status code --- 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 DEBUG --- status code --- 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG --- status code --- 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG --- status code --- 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:53 INFO [http_cmd]: http status code OK 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:53 DEBUG { 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG --- status code --- 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG { 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG { 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG { 19:39:53 DEBUG { 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG { 19:39:53 DEBUG { 19:39:53 DEBUG { 19:39:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:39:53 DEBUG { 19:39:53 DEBUG { 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG { 19:39:53 DEBUG { 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG { 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG { 19:39:53 DEBUG "data": { 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG { 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG --- http response --- 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "data": { 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG { 19:39:53 DEBUG { 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "data": { 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "data": { 19:39:53 DEBUG "data": { 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "data": { 19:39:53 DEBUG "data": { 19:39:53 DEBUG "data": { 19:39:53 DEBUG "data": { 19:39:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:39:53 DEBUG "data": { 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "data": { 19:39:53 DEBUG "data": { 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG "data": { 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "data": { 19:39:53 DEBUG "result": [ 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "data": { 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG ----- output ----- 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "result": [ 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "data": { 19:39:53 DEBUG "data": { 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "result": [ 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG { 19:39:53 DEBUG "result": [ 19:39:53 DEBUG "result": [ 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "result": [ 19:39:53 DEBUG "result": [ 19:39:53 DEBUG { 19:39:53 DEBUG "result": [ 19:39:53 DEBUG { 19:39:53 DEBUG "result": [ 19:39:53 DEBUG { 19:39:53 DEBUG "result": [ 19:39:53 DEBUG "result": [ 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "result": [ 19:39:53 DEBUG { 19:39:53 DEBUG "result": [ 19:39:53 DEBUG "metric": { 19:39:53 DEBUG { 19:39:53 DEBUG { 19:39:53 DEBUG "result": [ 19:39:53 DEBUG { 19:39:53 DEBUG "result": [ 19:39:53 DEBUG "metric": { 19:39:53 DEBUG { 19:39:53 DEBUG "status": "success", 19:39:53 DEBUG { 19:39:53 DEBUG "metric": { 19:39:53 DEBUG { 19:39:53 DEBUG { 19:39:53 DEBUG "result": [ 19:39:53 DEBUG "result": [ 19:39:53 DEBUG { 19:39:53 DEBUG "metric": { 19:39:53 DEBUG { 19:39:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:39:53 DEBUG "metric": { 19:39:53 DEBUG "metric": { 19:39:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:39:53 DEBUG "metric": { 19:39:53 DEBUG { 19:39:53 DEBUG "pod": "ds-cts-0" 19:39:53 DEBUG "metric": { 19:39:53 DEBUG "data": { 19:39:53 DEBUG "metric": { 19:39:53 DEBUG "pod": "ds-cts-0" 19:39:53 DEBUG "metric": { 19:39:53 DEBUG "metric": { 19:39:53 DEBUG { 19:39:53 DEBUG { 19:39:53 DEBUG "metric": { 19:39:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:39:53 DEBUG "metric": { 19:39:53 DEBUG }, 19:39:53 DEBUG "pod": "ds-idrepo-0" 19:39:53 DEBUG }, 19:39:53 DEBUG { 19:39:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:39:53 DEBUG "metric": { 19:39:53 DEBUG }, 19:39:53 DEBUG "instance": "10.142.0.107:9100" 19:39:53 DEBUG "resultType": "vector", 19:39:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:53 DEBUG }, 19:39:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:53 DEBUG "pod": "ds-idrepo-0" 19:39:53 DEBUG "metric": { 19:39:53 DEBUG "metric": { 19:39:53 DEBUG "instance": "10.142.0.107:9100" 19:39:53 DEBUG }, 19:39:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:53 DEBUG "value": [ 19:39:53 DEBUG }, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "metric": { 19:39:53 DEBUG }, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "value": [ 19:39:53 DEBUG }, 19:39:53 DEBUG "result": [ 19:39:53 DEBUG }, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG }, 19:39:53 DEBUG }, 19:39:53 DEBUG "instance": "10.142.0.107:9100" 19:39:53 DEBUG "instance": "10.142.0.107:9100" 19:39:53 DEBUG }, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG }, 19:39:53 DEBUG 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 19:39:53 DEBUG "pod": "ds-cts-0" 19:39:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG "value": [ 19:39:53 DEBUG { 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "value": [ 19:39:53 DEBUG }, 19:39:53 DEBUG }, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG }, 19:39:53 DEBUG }, 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG "metric": { 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG }, 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG "value": [ 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:39:53 DEBUG "pod": "ds-cts-2" 19:39:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:39:53 DEBUG [...] 19:39:53 DEBUG 19:39:53 DEBUG "pod": "ds-cts-2" 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG [...] 19:39:53 DEBUG "pod": "ds-cts-2" 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:39:53 DEBUG 19:39:53 DEBUG }, 19:39:53 DEBUG }, 19:39:53 DEBUG }, 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG }, 19:39:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:39:53 DEBUG "instance": "10.142.0.43:9100" 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG }, 19:39:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:53 DEBUG "pod": "ds-idrepo-2" 19:39:53 DEBUG 19:39:53 DEBUG 19:39:53 DEBUG "instance": "10.142.0.43:9100" 19:39:53 DEBUG }, 19:39:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:53 DEBUG }, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 19:39:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:53 DEBUG "value": [ 19:39:53 DEBUG }, 19:39:53 DEBUG }, 19:39:53 DEBUG 19:39:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:53 DEBUG "value": [ 19:39:53 DEBUG }, 19:39:53 DEBUG }, 19:39:53 DEBUG "instance": "10.142.0.43:9100" 19:39:53 DEBUG "instance": "10.142.0.43:9100" 19:39:53 DEBUG }, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG 19:39:53 DEBUG }, 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 19:39:53 DEBUG }, 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "value": [ 19:39:53 DEBUG }, 19:39:53 DEBUG }, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG "858730496" 19:39:53 DEBUG "0" 19:39:53 DEBUG "pod": "ds-cts-2" 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "0.0003985866666666666" 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "0" 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "0.17529999999997206" 19:39:53 DEBUG "0" 19:39:53 DEBUG "0" 19:39:53 DEBUG ] 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG ] 19:39:53 DEBUG }, 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG ] 19:39:53 DEBUG "0" 19:39:53 DEBUG "0.07099999999999226" 19:39:53 DEBUG }, 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG ] 19:39:53 DEBUG "0.154" 19:39:53 DEBUG "0" 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG ] 19:39:53 DEBUG ] 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG "0" 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG "2668.4666666666667" 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG ] 19:39:53 DEBUG "value": [ 19:39:53 DEBUG "479928.5" 19:39:53 DEBUG } 19:39:53 DEBUG "value": [ 19:39:53 DEBUG ] 19:39:53 DEBUG "81035791.16666666" 19:39:53 DEBUG "81031473.73333332" 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG ] 19:39:53 DEBUG ] 19:39:53 DEBUG ] 19:39:53 DEBUG 1713638393, 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG ] 19:39:53 DEBUG "2669.4333333333334" 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG "29.966666666666665" 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG ] 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG ] 19:39:53 DEBUG } 19:39:53 DEBUG } 19:39:53 DEBUG } 19:40:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638451" 19:40:51 ERROR [http_cmd]: ERROR 19:40:51 ERROR --- status code --- 19:40:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:40:51 ERROR --- http response --- 19:40:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:40:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:40:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638451" 19:40:51 INFO [http_cmd]: http status code OK 19:40:51 DEBUG --- status code --- 19:40:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:51 DEBUG --- http response --- 19:40:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:40:52 INFO 19:40:52 INFO [loop_until]: kubectl --namespace=xlou top pods 19:40:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:40:52 INFO [loop_until]: OK (rc = 0) 19:40:52 DEBUG --- stdout --- 19:40:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17108m 8654Mi am-767f6b67b5-qcqz2 16802m 9060Mi am-767f6b67b5-vbltz 19330m 8665Mi ds-cts-0 19013m 10260Mi ds-cts-1 17431m 10184Mi ds-cts-2 19559m 10239Mi ds-idrepo-0 7224m 3888Mi ds-idrepo-1 7217m 3905Mi ds-idrepo-2 7391m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8958m 729Mi 19:40:52 DEBUG --- stderr --- 19:40:52 DEBUG 19:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638452" 19:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638452" 19:40:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638452" 19:40:52 INFO [http_cmd]: http status code OK 19:40:52 DEBUG --- status code --- 19:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:52 DEBUG --- http response --- 19:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:52 DEBUG ----- output ----- 19:40:52 DEBUG { 19:40:52 DEBUG "status": "success", 19:40:52 DEBUG "data": { 19:40:52 DEBUG "resultType": "vector", 19:40:52 DEBUG "result": [ 19:40:52 DEBUG { 19:40:52 DEBUG "metric": { 19:40:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:52 DEBUG }, 19:40:52 DEBUG "value": [ 19:40:52 DEBUG 19:40:52 DEBUG 19:40:52 DEBUG 19:40:52 DEBUG [...] 19:40:52 DEBUG 19:40:52 DEBUG 19:40:52 DEBUG 19:40:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:52 DEBUG }, 19:40:52 DEBUG "value": [ 19:40:52 DEBUG 1713638452, 19:40:52 DEBUG "2666.9000000000005" 19:40:52 DEBUG ] 19:40:52 DEBUG } 19:40:52 DEBUG ] 19:40:52 DEBUG } 19:40:52 DEBUG } 19:40:52 INFO [http_cmd]: http status code OK 19:40:52 DEBUG --- status code --- 19:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:52 DEBUG --- http response --- 19:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:52 DEBUG ----- output ----- 19:40:52 DEBUG { 19:40:52 DEBUG "status": "success", 19:40:52 DEBUG "data": { 19:40:52 DEBUG "resultType": "vector", 19:40:52 DEBUG "result": [ 19:40:52 DEBUG { 19:40:52 DEBUG "metric": { 19:40:52 DEBUG "pod": "ds-idrepo-0" 19:40:52 DEBUG }, 19:40:52 DEBUG "value": [ 19:40:52 DEBUG 19:40:52 DEBUG 19:40:52 DEBUG 19:40:52 DEBUG [...] 19:40:52 DEBUG 19:40:52 DEBUG 19:40:52 DEBUG 19:40:52 DEBUG "pod": "ds-idrepo-2" 19:40:52 DEBUG }, 19:40:52 DEBUG "value": [ 19:40:52 DEBUG 1713638452, 19:40:52 DEBUG "0" 19:40:52 DEBUG ] 19:40:52 DEBUG } 19:40:52 DEBUG ] 19:40:52 DEBUG } 19:40:52 DEBUG } 19:40:52 INFO [http_cmd]: http status code OK 19:40:52 DEBUG --- status code --- 19:40:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:52 DEBUG --- http response --- 19:40:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:52 DEBUG ----- output ----- 19:40:52 DEBUG { 19:40:52 DEBUG "status": "success", 19:40:52 DEBUG "data": { 19:40:52 DEBUG "resultType": "vector", 19:40:52 DEBUG "result": [ 19:40:52 DEBUG { 19:40:52 DEBUG "metric": { 19:40:52 DEBUG "pod": "ds-idrepo-0" 19:40:52 DEBUG }, 19:40:52 DEBUG "value": [ 19:40:52 DEBUG 19:40:52 DEBUG 19:40:52 DEBUG 19:40:52 DEBUG [...] 19:40:52 DEBUG 19:40:52 DEBUG 19:40:52 DEBUG 19:40:52 DEBUG "pod": "ds-idrepo-2" 19:40:52 DEBUG }, 19:40:52 DEBUG "value": [ 19:40:52 DEBUG 1713638452, 19:40:52 DEBUG "0" 19:40:52 DEBUG ] 19:40:52 DEBUG } 19:40:52 DEBUG ] 19:40:52 DEBUG } 19:40:52 DEBUG } 19:40:53 INFO 19:40:53 INFO [loop_until]: kubectl --namespace=xlou top node 19:40:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:40:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638453" 19:40:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638453" 19:40:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638453" 19:40:53 INFO [http_cmd]: http status code OK 19:40:53 DEBUG --- status code --- 19:40:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:53 DEBUG --- http response --- 19:40:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:53 DEBUG ----- output ----- 19:40:53 DEBUG { 19:40:53 DEBUG "status": "success", 19:40:53 INFO [http_cmd]: http status code OK 19:40:53 DEBUG "data": { 19:40:53 DEBUG --- status code --- 19:40:53 DEBUG "resultType": "vector", 19:40:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:53 DEBUG "result": [ 19:40:53 DEBUG --- http response --- 19:40:53 DEBUG { 19:40:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:53 DEBUG ----- output ----- 19:40:53 INFO [http_cmd]: http status code OK 19:40:53 DEBUG "metric": { 19:40:53 DEBUG { 19:40:53 DEBUG --- status code --- 19:40:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:53 INFO [loop_until]: OK (rc = 0) 19:40:53 DEBUG "status": "success", 19:40:53 DEBUG --- http response --- 19:40:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:53 DEBUG --- stdout --- 19:40:53 DEBUG "data": { 19:40:53 DEBUG }, 19:40:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:40:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6214Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7350m 24% 7633Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16482m 55% 11375Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 92m 0% 3363Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19193m 64% 13486Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5195Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17383m 58% 11939Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14668m 49% 2947Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19448m 65% 10884Mi 9% gke-xlou-cdm-ds-0f601751-j488 8224m 27% 7189Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17347m 58% 15266Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8106m 27% 7882Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18817m 63% 12934Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9145m 57% 6450Mi 10% 19:40:53 DEBUG "resultType": "vector", 19:40:53 DEBUG "value": [ 19:40:53 DEBUG ----- output ----- 19:40:53 DEBUG --- stderr --- 19:40:53 DEBUG "result": [ 19:40:53 DEBUG 19:40:53 DEBUG { 19:40:53 DEBUG 19:40:53 DEBUG { 19:40:53 DEBUG 19:40:53 DEBUG "status": "success", 19:40:53 DEBUG "metric": { 19:40:53 DEBUG 19:40:53 DEBUG [...] 19:40:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:40:53 DEBUG 19:40:53 DEBUG "data": { 19:40:53 DEBUG }, 19:40:53 DEBUG 19:40:53 DEBUG "resultType": "vector", 19:40:53 DEBUG "value": [ 19:40:53 DEBUG 19:40:53 DEBUG "result": [ 19:40:53 DEBUG 19:40:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:53 DEBUG }, 19:40:53 DEBUG { 19:40:53 DEBUG 19:40:53 DEBUG "value": [ 19:40:53 DEBUG 1713638453, 19:40:53 DEBUG 19:40:53 DEBUG "metric": { 19:40:53 DEBUG "2664.5666666666666" 19:40:53 DEBUG [...] 19:40:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:40:53 DEBUG ] 19:40:53 DEBUG 19:40:53 DEBUG }, 19:40:53 DEBUG } 19:40:53 DEBUG 19:40:53 DEBUG "value": [ 19:40:53 DEBUG ] 19:40:53 DEBUG 19:40:53 DEBUG 19:40:53 DEBUG } 19:40:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:40:53 DEBUG 19:40:53 DEBUG } 19:40:53 DEBUG }, 19:40:53 DEBUG 19:40:53 DEBUG "value": [ 19:40:53 DEBUG [...] 19:40:53 DEBUG 19:40:53 DEBUG 19:40:53 DEBUG 19:40:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:40:53 DEBUG 1713638453, 19:40:53 DEBUG }, 19:40:53 DEBUG "8.953155938030358" 19:40:53 DEBUG "value": [ 19:40:53 DEBUG ] 19:40:53 DEBUG 1713638453, 19:40:53 DEBUG } 19:40:53 DEBUG "0" 19:40:53 DEBUG ] 19:40:53 DEBUG ] 19:40:53 DEBUG } 19:40:53 DEBUG } 19:40:53 DEBUG } 19:40:53 DEBUG ] 19:40:53 DEBUG } 19:40:53 DEBUG } 19:40:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638453" 19:40:53 INFO [http_cmd]: http status code OK 19:40:53 DEBUG --- status code --- 19:40:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:53 DEBUG --- http response --- 19:40:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:53 DEBUG ----- output ----- 19:40:53 DEBUG { 19:40:53 DEBUG "status": "success", 19:40:53 DEBUG "data": { 19:40:53 DEBUG "resultType": "vector", 19:40:53 DEBUG "result": [ 19:40:53 DEBUG { 19:40:53 DEBUG "metric": { 19:40:53 DEBUG "instance": "10.142.0.107:9100" 19:40:53 DEBUG }, 19:40:53 DEBUG "value": [ 19:40:53 DEBUG 19:40:53 DEBUG 19:40:53 DEBUG 19:40:53 DEBUG [...] 19:40:53 DEBUG 19:40:53 DEBUG 19:40:53 DEBUG 19:40:53 DEBUG "instance": "10.142.0.43:9100" 19:40:53 DEBUG }, 19:40:53 DEBUG "value": [ 19:40:53 DEBUG 1713638453, 19:40:53 DEBUG "3569759249408" 19:40:53 DEBUG ] 19:40:53 DEBUG } 19:40:53 DEBUG ] 19:40:53 DEBUG } 19:40:53 DEBUG } 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638454" 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 DEBUG --- status code --- 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG --- status code --- 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG --- status code --- 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 INFO [http_cmd]: http status code OK 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG --- status code --- 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:40:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG { 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:54 DEBUG { 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG { 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG --- http response --- 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:40:54 DEBUG { 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG { 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG { 19:40:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:40:54 DEBUG { 19:40:54 DEBUG { 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG { 19:40:54 DEBUG "data": { 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG { 19:40:54 DEBUG { 19:40:54 DEBUG "data": { 19:40:54 DEBUG { 19:40:54 DEBUG { 19:40:54 DEBUG { 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "data": { 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "data": { 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "data": { 19:40:54 DEBUG { 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG { 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG { 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "data": { 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "data": { 19:40:54 DEBUG ----- output ----- 19:40:54 DEBUG { 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "data": { 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "data": { 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "data": { 19:40:54 DEBUG "data": { 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "data": { 19:40:54 DEBUG "data": { 19:40:54 DEBUG "data": { 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "status": "success", 19:40:54 DEBUG "data": { 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "data": { 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG { 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG { 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "result": [ 19:40:54 DEBUG { 19:40:54 DEBUG { 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "data": { 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:40:54 DEBUG "data": { 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "result": [ 19:40:54 DEBUG { 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "metric": { 19:40:54 DEBUG { 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG "result": [ 19:40:54 DEBUG { 19:40:54 DEBUG { 19:40:54 DEBUG "result": [ 19:40:54 DEBUG { 19:40:54 DEBUG }, 19:40:54 DEBUG "resultType": "vector", 19:40:54 DEBUG { 19:40:54 DEBUG { 19:40:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:40:54 DEBUG }, 19:40:54 DEBUG { 19:40:54 DEBUG { 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "instance": "10.142.0.107:9100" 19:40:54 DEBUG "pod": "ds-cts-0" 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "result": [ 19:40:54 DEBUG { 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "metric": { 19:40:54 DEBUG { 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "value": [ 19:40:54 DEBUG "result": [ 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "metric": { 19:40:54 DEBUG { 19:40:54 DEBUG "value": [ 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:54 DEBUG }, 19:40:54 DEBUG }, 19:40:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:54 DEBUG { 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "pod": "ds-cts-0" 19:40:54 DEBUG "instance": "10.142.0.107:9100" 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:40:54 DEBUG 19:40:54 DEBUG { 19:40:54 DEBUG "pod": "ds-idrepo-0" 19:40:54 DEBUG "instance": "10.142.0.107:9100" 19:40:54 DEBUG "metric": { 19:40:54 DEBUG 19:40:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:40:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:54 DEBUG }, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG "value": [ 19:40:54 DEBUG }, 19:40:54 DEBUG "metric": { 19:40:54 DEBUG "pod": "ds-cts-0" 19:40:54 DEBUG }, 19:40:54 DEBUG }, 19:40:54 DEBUG "instance": "10.142.0.107:9100" 19:40:54 DEBUG }, 19:40:54 DEBUG 19:40:54 DEBUG "metric": { 19:40:54 DEBUG }, 19:40:54 DEBUG }, 19:40:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:54 DEBUG 19:40:54 DEBUG }, 19:40:54 DEBUG }, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG "value": [ 19:40:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:40:54 DEBUG }, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG "value": [ 19:40:54 DEBUG }, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 19:40:54 DEBUG "pod": "ds-idrepo-0" 19:40:54 DEBUG "value": [ 19:40:54 DEBUG "value": [ 19:40:54 DEBUG }, 19:40:54 DEBUG 19:40:54 DEBUG "value": [ 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG }, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 19:40:54 DEBUG [...] 19:40:54 DEBUG }, 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG "value": [ 19:40:54 DEBUG [...] 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG [...] 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG [...] 19:40:54 DEBUG [...] 19:40:54 DEBUG [...] 19:40:54 DEBUG 19:40:54 DEBUG [...] 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG [...] 19:40:54 DEBUG [...] 19:40:54 DEBUG 19:40:54 DEBUG [...] 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG [...] 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG [...] 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG [...] 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG [...] 19:40:54 DEBUG 19:40:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:40:54 DEBUG [...] 19:40:54 DEBUG 19:40:54 DEBUG [...] 19:40:54 DEBUG [...] 19:40:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG [...] 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG }, 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG }, 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG "pod": "ds-cts-2" 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG "instance": "10.142.0.43:9100" 19:40:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:54 DEBUG }, 19:40:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:54 DEBUG 19:40:54 DEBUG "pod": "ds-cts-2" 19:40:54 DEBUG 19:40:54 DEBUG "instance": "10.142.0.43:9100" 19:40:54 DEBUG 19:40:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG 19:40:54 DEBUG "instance": "10.142.0.43:9100" 19:40:54 DEBUG 19:40:54 DEBUG 19:40:54 DEBUG "17.809196281159398" 19:40:54 DEBUG 19:40:54 DEBUG }, 19:40:54 DEBUG }, 19:40:54 DEBUG }, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG }, 19:40:54 DEBUG 19:40:54 DEBUG }, 19:40:54 DEBUG "pod": "ds-cts-2" 19:40:54 DEBUG }, 19:40:54 DEBUG "instance": "10.142.0.43:9100" 19:40:54 DEBUG }, 19:40:54 DEBUG "863363072" 19:40:54 DEBUG "pod": "ds-idrepo-2" 19:40:54 DEBUG }, 19:40:54 DEBUG "pod": "ds-cts-2" 19:40:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:54 DEBUG ] 19:40:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:40:54 DEBUG "value": [ 19:40:54 DEBUG "value": [ 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:40:54 DEBUG "value": [ 19:40:54 DEBUG }, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG }, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG ] 19:40:54 DEBUG }, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG }, 19:40:54 DEBUG } 19:40:54 DEBUG }, 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG "2668.3333333333335" 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG "0.154" 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG } 19:40:54 DEBUG "value": [ 19:40:54 DEBUG }, 19:40:54 DEBUG "81060159.36666667" 19:40:54 DEBUG "value": [ 19:40:54 DEBUG ] 19:40:54 DEBUG "value": [ 19:40:54 DEBUG "0.07233333333333765" 19:40:54 DEBUG "0" 19:40:54 DEBUG ] 19:40:54 DEBUG "0" 19:40:54 DEBUG ] 19:40:54 DEBUG "29.9" 19:40:54 DEBUG }, 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG "0.17649999999995408" 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG "116190.99469018992" 19:40:54 DEBUG ] 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG "value": [ 19:40:54 DEBUG ] 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG } 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG ] 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG "81056480.13333333" 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG "0" 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG } 19:40:54 DEBUG "480098" 19:40:54 DEBUG } 19:40:54 DEBUG "14651.170367665836" 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG "0.00048595999999999996" 19:40:54 DEBUG "value": [ 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG ] 19:40:54 DEBUG "0" 19:40:54 DEBUG ] 19:40:54 DEBUG ] 19:40:54 DEBUG ] 19:40:54 DEBUG ] 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG ] 19:40:54 DEBUG 1713638454, 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG "2668.5666666666666" 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG ] 19:40:54 DEBUG } 19:40:54 DEBUG } 19:40:54 DEBUG } 19:41:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638511" 19:41:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638511" 19:41:51 ERROR [http_cmd]: ERROR 19:41:51 ERROR --- status code --- 19:41:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:41:51 ERROR --- http response --- 19:41:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:41:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:41:51 INFO [http_cmd]: http status code OK 19:41:51 DEBUG --- status code --- 19:41:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:51 DEBUG --- http response --- 19:41:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:41:52 INFO 19:41:52 INFO [loop_until]: kubectl --namespace=xlou top pods 19:41:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638512" 19:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638512" 19:41:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638512" 19:41:52 INFO [http_cmd]: http status code OK 19:41:52 INFO [http_cmd]: http status code OK 19:41:52 DEBUG --- status code --- 19:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:52 DEBUG --- http response --- 19:41:52 DEBUG --- status code --- 19:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:52 DEBUG --- http response --- 19:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:52 DEBUG ----- output ----- 19:41:52 DEBUG { 19:41:52 DEBUG "status": "success", 19:41:52 DEBUG "data": { 19:41:52 DEBUG "resultType": "vector", 19:41:52 DEBUG "result": [ 19:41:52 DEBUG { 19:41:52 DEBUG "metric": { 19:41:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:52 DEBUG }, 19:41:52 DEBUG "value": [ 19:41:52 DEBUG 19:41:52 DEBUG 19:41:52 DEBUG 19:41:52 DEBUG [...] 19:41:52 DEBUG 19:41:52 DEBUG 19:41:52 DEBUG 19:41:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:52 DEBUG }, 19:41:52 DEBUG "value": [ 19:41:52 DEBUG 1713638512, 19:41:52 DEBUG "2663.333333333334" 19:41:52 DEBUG ] 19:41:52 DEBUG } 19:41:52 DEBUG ] 19:41:52 DEBUG } 19:41:52 DEBUG } 19:41:52 DEBUG ----- output ----- 19:41:52 DEBUG { 19:41:52 DEBUG "status": "success", 19:41:52 DEBUG "data": { 19:41:52 DEBUG "resultType": "vector", 19:41:52 DEBUG "result": [ 19:41:52 DEBUG { 19:41:52 DEBUG "metric": { 19:41:52 DEBUG "pod": "ds-idrepo-0" 19:41:52 DEBUG }, 19:41:52 DEBUG "value": [ 19:41:52 DEBUG 19:41:52 DEBUG 19:41:52 DEBUG 19:41:52 DEBUG [...] 19:41:52 DEBUG 19:41:52 DEBUG 19:41:52 DEBUG 19:41:52 DEBUG "pod": "ds-idrepo-2" 19:41:52 DEBUG }, 19:41:52 DEBUG "value": [ 19:41:52 DEBUG 1713638512, 19:41:52 DEBUG "0" 19:41:52 DEBUG ] 19:41:52 DEBUG } 19:41:52 DEBUG ] 19:41:52 DEBUG } 19:41:52 DEBUG } 19:41:52 INFO [http_cmd]: http status code OK 19:41:52 DEBUG --- status code --- 19:41:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:52 DEBUG --- http response --- 19:41:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:52 DEBUG ----- output ----- 19:41:52 DEBUG { 19:41:52 DEBUG "status": "success", 19:41:52 DEBUG "data": { 19:41:52 DEBUG "resultType": "vector", 19:41:52 DEBUG "result": [ 19:41:52 DEBUG { 19:41:52 DEBUG "metric": { 19:41:52 DEBUG "pod": "ds-idrepo-0" 19:41:52 DEBUG }, 19:41:52 DEBUG "value": [ 19:41:52 DEBUG 19:41:52 DEBUG 19:41:52 DEBUG 19:41:52 DEBUG [...] 19:41:52 DEBUG 19:41:52 DEBUG 19:41:52 DEBUG 19:41:52 DEBUG "pod": "ds-idrepo-2" 19:41:52 DEBUG }, 19:41:52 DEBUG "value": [ 19:41:52 DEBUG 1713638512, 19:41:52 DEBUG "0" 19:41:52 DEBUG ] 19:41:52 DEBUG } 19:41:52 DEBUG ] 19:41:52 DEBUG } 19:41:52 DEBUG } 19:41:52 INFO [loop_until]: OK (rc = 0) 19:41:52 DEBUG --- stdout --- 19:41:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16984m 8646Mi am-767f6b67b5-qcqz2 16709m 9061Mi am-767f6b67b5-vbltz 18650m 8667Mi ds-cts-0 18877m 10239Mi ds-cts-1 17308m 10188Mi ds-cts-2 18651m 10251Mi ds-idrepo-0 7333m 3888Mi ds-idrepo-1 7139m 3906Mi ds-idrepo-2 7535m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 16m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8852m 730Mi 19:41:52 DEBUG --- stderr --- 19:41:52 DEBUG 19:41:53 INFO 19:41:53 INFO [loop_until]: kubectl --namespace=xlou top node 19:41:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:41:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638513" 19:41:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638513" 19:41:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638513" 19:41:53 INFO [http_cmd]: http status code OK 19:41:53 DEBUG --- status code --- 19:41:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:53 DEBUG --- http response --- 19:41:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:53 DEBUG ----- output ----- 19:41:53 DEBUG { 19:41:53 DEBUG "status": "success", 19:41:53 DEBUG "data": { 19:41:53 DEBUG "resultType": "vector", 19:41:53 DEBUG "result": [ 19:41:53 DEBUG { 19:41:53 DEBUG "metric": { 19:41:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:53 DEBUG }, 19:41:53 DEBUG "value": [ 19:41:53 DEBUG 19:41:53 DEBUG 19:41:53 DEBUG 19:41:53 DEBUG [...] 19:41:53 DEBUG 19:41:53 DEBUG 19:41:53 DEBUG 19:41:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:53 DEBUG }, 19:41:53 DEBUG "value": [ 19:41:53 DEBUG 1713638513, 19:41:53 DEBUG "2661.1" 19:41:53 DEBUG ] 19:41:53 DEBUG } 19:41:53 DEBUG ] 19:41:53 DEBUG } 19:41:53 DEBUG } 19:41:53 INFO [http_cmd]: http status code OK 19:41:53 DEBUG --- status code --- 19:41:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:53 DEBUG --- http response --- 19:41:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:41:53 DEBUG ----- output ----- 19:41:53 DEBUG { 19:41:53 DEBUG "status": "success", 19:41:53 DEBUG "data": { 19:41:53 INFO [http_cmd]: http status code OK 19:41:53 DEBUG "resultType": "vector", 19:41:53 DEBUG --- status code --- 19:41:53 DEBUG "result": [ 19:41:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:53 DEBUG { 19:41:53 DEBUG --- http response --- 19:41:53 DEBUG "metric": { 19:41:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:41:53 DEBUG ----- output ----- 19:41:53 DEBUG }, 19:41:53 DEBUG { 19:41:53 DEBUG "value": [ 19:41:53 DEBUG "status": "success", 19:41:53 DEBUG 19:41:53 DEBUG "data": { 19:41:53 DEBUG 19:41:53 DEBUG "resultType": "vector", 19:41:53 DEBUG 19:41:53 DEBUG "result": [ 19:41:53 DEBUG [...] 19:41:53 DEBUG { 19:41:53 DEBUG 19:41:53 DEBUG "metric": { 19:41:53 DEBUG 19:41:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:41:53 DEBUG 19:41:53 DEBUG "pod": "ds-cts-0" 19:41:53 DEBUG }, 19:41:53 DEBUG "value": [ 19:41:53 DEBUG }, 19:41:53 DEBUG 1713638513, 19:41:53 DEBUG "value": [ 19:41:53 DEBUG 19:41:53 DEBUG "77568333.29613511" 19:41:53 DEBUG 19:41:53 DEBUG ] 19:41:53 DEBUG 19:41:53 DEBUG } 19:41:53 DEBUG [...] 19:41:53 DEBUG ] 19:41:53 DEBUG 19:41:53 DEBUG } 19:41:53 DEBUG 19:41:53 DEBUG } 19:41:53 DEBUG 19:41:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:41:53 DEBUG }, 19:41:53 DEBUG "value": [ 19:41:53 DEBUG 1713638513, 19:41:53 DEBUG "8.892181774208511" 19:41:53 DEBUG ] 19:41:53 DEBUG } 19:41:53 DEBUG ] 19:41:53 DEBUG } 19:41:53 DEBUG } 19:41:53 INFO [loop_until]: OK (rc = 0) 19:41:53 DEBUG --- stdout --- 19:41:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6213Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7545m 25% 7630Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16817m 56% 11365Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 91m 0% 3357Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19313m 64% 13477Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5182Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17232m 57% 11952Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14590m 48% 2948Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19559m 65% 10890Mi 9% gke-xlou-cdm-ds-0f601751-j488 8148m 27% 7189Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17349m 58% 15272Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8102m 27% 7861Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18846m 63% 12930Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9076m 57% 6450Mi 10% 19:41:53 DEBUG --- stderr --- 19:41:53 DEBUG 19:41:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638513" 19:41:53 INFO [http_cmd]: http status code OK 19:41:53 DEBUG --- status code --- 19:41:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:53 DEBUG --- http response --- 19:41:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:53 DEBUG ----- output ----- 19:41:53 DEBUG { 19:41:53 DEBUG "status": "success", 19:41:53 DEBUG "data": { 19:41:53 DEBUG "resultType": "vector", 19:41:53 DEBUG "result": [ 19:41:53 DEBUG { 19:41:53 DEBUG "metric": { 19:41:53 DEBUG "instance": "10.142.0.107:9100" 19:41:53 DEBUG }, 19:41:53 DEBUG "value": [ 19:41:53 DEBUG 19:41:53 DEBUG 19:41:53 DEBUG 19:41:53 DEBUG [...] 19:41:53 DEBUG 19:41:53 DEBUG 19:41:53 DEBUG 19:41:53 DEBUG "instance": "10.142.0.43:9100" 19:41:53 DEBUG }, 19:41:53 DEBUG "value": [ 19:41:53 DEBUG 1713638513, 19:41:53 DEBUG "3574002150400" 19:41:53 DEBUG ] 19:41:53 DEBUG } 19:41:53 DEBUG ] 19:41:53 DEBUG } 19:41:53 DEBUG } 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638514" 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 DEBUG --- status code --- 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 DEBUG --- status code --- 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG --- http response --- 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 DEBUG --- status code --- 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 DEBUG --- status code --- 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:54 DEBUG ----- output ----- 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG { 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "data": { 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG --- http response --- 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 INFO [http_cmd]: http status code OK 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG { 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG --- status code --- 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG { 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:41:54 DEBUG "result": [ 19:41:54 DEBUG { 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG { 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG { 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG { 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "metric": { 19:41:54 DEBUG { 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "data": { 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG { 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG { 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "data": { 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "data": { 19:41:54 DEBUG "data": { 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:54 DEBUG }, 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "data": { 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG "result": [ 19:41:54 DEBUG { 19:41:54 DEBUG { 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "data": { 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG "result": [ 19:41:54 DEBUG { 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG { 19:41:54 DEBUG "value": [ 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG "result": [ 19:41:54 DEBUG "metric": { 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG { 19:41:54 DEBUG { 19:41:54 DEBUG "data": { 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG --- http response --- 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG "metric": { 19:41:54 DEBUG { 19:41:54 DEBUG "result": [ 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG 19:41:54 DEBUG { 19:41:54 DEBUG { 19:41:54 DEBUG "pod": "ds-idrepo-0" 19:41:54 DEBUG "data": { 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "result": [ 19:41:54 DEBUG "data": { 19:41:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG "result": [ 19:41:54 DEBUG { 19:41:54 DEBUG "instance": "10.142.0.107:9100" 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "data": { 19:41:54 DEBUG "data": { 19:41:54 DEBUG "result": [ 19:41:54 DEBUG 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "metric": { 19:41:54 DEBUG }, 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG "data": { 19:41:54 DEBUG "data": { 19:41:54 DEBUG { 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG ----- output ----- 19:41:54 DEBUG { 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG }, 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG { 19:41:54 DEBUG "metric": { 19:41:54 DEBUG { 19:41:54 DEBUG 19:41:54 DEBUG "data": { 19:41:54 DEBUG "instance": "10.142.0.107:9100" 19:41:54 DEBUG "value": [ 19:41:54 DEBUG "result": [ 19:41:54 DEBUG { 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG "result": [ 19:41:54 DEBUG "result": [ 19:41:54 DEBUG { 19:41:54 DEBUG "data": { 19:41:54 DEBUG { 19:41:54 DEBUG "metric": { 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 19:41:54 DEBUG "instance": "10.142.0.107:9100" 19:41:54 DEBUG "result": [ 19:41:54 DEBUG "metric": { 19:41:54 DEBUG "pod": "ds-cts-0" 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG "metric": { 19:41:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:54 DEBUG "metric": { 19:41:54 DEBUG { 19:41:54 DEBUG "status": "success", 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:41:54 DEBUG }, 19:41:54 DEBUG "value": [ 19:41:54 DEBUG }, 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG "result": [ 19:41:54 DEBUG }, 19:41:54 DEBUG [...] 19:41:54 DEBUG "result": [ 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 19:41:54 DEBUG "result": [ 19:41:54 DEBUG }, 19:41:54 DEBUG { 19:41:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:54 DEBUG "metric": { 19:41:54 DEBUG "data": { 19:41:54 DEBUG "result": [ 19:41:54 DEBUG "data": { 19:41:54 DEBUG 19:41:54 DEBUG "value": [ 19:41:54 DEBUG { 19:41:54 DEBUG { 19:41:54 DEBUG 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG { 19:41:54 DEBUG "metric": { 19:41:54 DEBUG "metric": { 19:41:54 DEBUG }, 19:41:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG { 19:41:54 DEBUG "resultType": "vector", 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG [...] 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG { 19:41:54 DEBUG 19:41:54 DEBUG [...] 19:41:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:41:54 DEBUG "value": [ 19:41:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:54 DEBUG "value": [ 19:41:54 DEBUG }, 19:41:54 DEBUG "result": [ 19:41:54 DEBUG "metric": { 19:41:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG "metric": { 19:41:54 DEBUG "metric": { 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:54 DEBUG "metric": { 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG "value": [ 19:41:54 DEBUG { 19:41:54 DEBUG "value": [ 19:41:54 DEBUG "result": [ 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG "pod": "ds-cts-0" 19:41:54 DEBUG "pod": "ds-idrepo-0" 19:41:54 DEBUG [...] 19:41:54 DEBUG 19:41:54 DEBUG }, 19:41:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:41:54 DEBUG [...] 19:41:54 DEBUG 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG { 19:41:54 DEBUG [...] 19:41:54 DEBUG 19:41:54 DEBUG }, 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 19:41:54 DEBUG [...] 19:41:54 DEBUG "value": [ 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG [...] 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG "metric": { 19:41:54 DEBUG 19:41:54 DEBUG "metric": { 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 19:41:54 DEBUG "instance": "10.142.0.43:9100" 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG [...] 19:41:54 DEBUG 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 19:41:54 DEBUG "pod": "ds-cts-2" 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG [...] 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG "pod": "ds-cts-0" 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG [...] 19:41:54 DEBUG 19:41:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:54 DEBUG "instance": "10.142.0.107:9100" 19:41:54 DEBUG }, 19:41:54 DEBUG [...] 19:41:54 DEBUG 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 19:41:54 DEBUG "2661.366666666667" 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG "instance": "10.142.0.43:9100" 19:41:54 DEBUG }, 19:41:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:54 DEBUG 19:41:54 DEBUG }, 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG "value": [ 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG } 19:41:54 DEBUG 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG "0" 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 19:41:54 DEBUG [...] 19:41:54 DEBUG 19:41:54 DEBUG "pod": "ds-cts-0" 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG "pod": "ds-cts-2" 19:41:54 DEBUG "pod": "ds-cts-2" 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG "0.06966666666665257" 19:41:54 DEBUG 19:41:54 DEBUG "80817060.03333333" 19:41:54 DEBUG [...] 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 19:41:54 DEBUG ] 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG "value": [ 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG }, 19:41:54 DEBUG "instance": "10.142.0.43:9100" 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG [...] 19:41:54 DEBUG ] 19:41:54 DEBUG 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG } 19:41:54 DEBUG "0" 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG [...] 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG "24.366666666666667" 19:41:54 DEBUG } 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG "pod": "ds-cts-0" 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG 19:41:54 DEBUG "pod": "ds-cts-0" 19:41:54 DEBUG ] 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG 19:41:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:54 DEBUG [...] 19:41:54 DEBUG 19:41:54 DEBUG "0" 19:41:54 DEBUG }, 19:41:54 DEBUG ] 19:41:54 DEBUG ] 19:41:54 DEBUG }, 19:41:54 DEBUG } 19:41:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:41:54 DEBUG "2661.366666666667" 19:41:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:41:54 DEBUG }, 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG 19:41:54 DEBUG }, 19:41:54 DEBUG "40828122.048045225" 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG "value": [ 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG "value": [ 19:41:54 DEBUG ] 19:41:54 DEBUG }, 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG "value": [ 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG 19:41:54 DEBUG "value": [ 19:41:54 DEBUG ] 19:41:54 DEBUG 19:41:54 DEBUG [...] 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG "80822309.36666667" 19:41:54 DEBUG } 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG "355881.5267938846" 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG "0.154" 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG "862973952" 19:41:54 DEBUG 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG "480479" 19:41:54 DEBUG 19:41:54 DEBUG 19:41:54 DEBUG ] 19:41:54 DEBUG ] 19:41:54 DEBUG ] 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG "pod": "ds-cts-2" 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG "pod": "ds-idrepo-2" 19:41:54 DEBUG 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG }, 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG }, 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG }, 19:41:54 DEBUG 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG "value": [ 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG "value": [ 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG "value": [ 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG "instance": "10.142.0.43:9100" 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG } 19:41:54 DEBUG "13.90781834239632" 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG "0.0002666666666666666" 19:41:54 DEBUG }, 19:41:54 DEBUG "0" 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG "value": [ 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG 1713638514, 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG "0.1813666666666696" 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG } 19:41:54 DEBUG ] 19:41:54 DEBUG } 19:41:54 DEBUG } 19:42:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638571" 19:42:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638571" 19:42:51 ERROR [http_cmd]: ERROR 19:42:51 ERROR --- status code --- 19:42:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:42:51 ERROR --- http response --- 19:42:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:42:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:42:51 INFO [http_cmd]: http status code OK 19:42:51 DEBUG --- status code --- 19:42:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:51 DEBUG --- http response --- 19:42:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638572" 19:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638572" 19:42:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638572" 19:42:52 INFO 19:42:52 INFO [http_cmd]: http status code OK 19:42:52 INFO [http_cmd]: http status code OK 19:42:52 INFO [loop_until]: kubectl --namespace=xlou top pods 19:42:52 DEBUG --- status code --- 19:42:52 DEBUG --- status code --- 19:42:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:52 DEBUG --- http response --- 19:42:52 DEBUG --- http response --- 19:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:52 DEBUG ----- output ----- 19:42:52 INFO [http_cmd]: http status code OK 19:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:52 DEBUG { 19:42:52 DEBUG --- status code --- 19:42:52 DEBUG ----- output ----- 19:42:52 DEBUG { 19:42:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:52 DEBUG "status": "success", 19:42:52 DEBUG "status": "success", 19:42:52 DEBUG --- http response --- 19:42:52 DEBUG "data": { 19:42:52 DEBUG "data": { 19:42:52 DEBUG "resultType": "vector", 19:42:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:52 DEBUG "resultType": "vector", 19:42:52 DEBUG "result": [ 19:42:52 DEBUG ----- output ----- 19:42:52 DEBUG "result": [ 19:42:52 DEBUG { 19:42:52 DEBUG { 19:42:52 DEBUG { 19:42:52 DEBUG "metric": { 19:42:52 DEBUG "status": "success", 19:42:52 DEBUG "metric": { 19:42:52 DEBUG "pod": "ds-idrepo-0" 19:42:52 DEBUG "data": { 19:42:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:52 DEBUG }, 19:42:52 DEBUG "resultType": "vector", 19:42:52 DEBUG }, 19:42:52 DEBUG "value": [ 19:42:52 DEBUG "result": [ 19:42:52 DEBUG "value": [ 19:42:52 DEBUG 19:42:52 DEBUG { 19:42:52 DEBUG 19:42:52 DEBUG 19:42:52 DEBUG "metric": { 19:42:52 DEBUG 19:42:52 DEBUG 19:42:52 DEBUG "pod": "ds-idrepo-0" 19:42:52 DEBUG 19:42:52 DEBUG [...] 19:42:52 DEBUG }, 19:42:52 DEBUG [...] 19:42:52 DEBUG 19:42:52 DEBUG "value": [ 19:42:52 DEBUG 19:42:52 DEBUG 19:42:52 DEBUG 19:42:52 DEBUG 19:42:52 DEBUG "pod": "ds-idrepo-2" 19:42:52 DEBUG 19:42:52 DEBUG 19:42:52 DEBUG }, 19:42:52 DEBUG 19:42:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:52 DEBUG "value": [ 19:42:52 DEBUG 19:42:52 DEBUG }, 19:42:52 DEBUG 1713638572, 19:42:52 DEBUG [...] 19:42:52 DEBUG "value": [ 19:42:52 DEBUG "0" 19:42:52 DEBUG 19:42:52 DEBUG 1713638572, 19:42:52 DEBUG ] 19:42:52 DEBUG 19:42:52 DEBUG "2675.0000000000005" 19:42:52 DEBUG ] 19:42:52 DEBUG 19:42:52 DEBUG } 19:42:52 DEBUG } 19:42:52 DEBUG "pod": "ds-idrepo-2" 19:42:52 DEBUG ] 19:42:52 DEBUG ] 19:42:52 DEBUG }, 19:42:52 DEBUG } 19:42:52 DEBUG } 19:42:52 DEBUG "value": [ 19:42:52 DEBUG } 19:42:52 DEBUG } 19:42:52 DEBUG 1713638572, 19:42:52 DEBUG "0" 19:42:52 DEBUG ] 19:42:52 DEBUG } 19:42:52 DEBUG ] 19:42:52 DEBUG } 19:42:52 DEBUG } 19:42:52 INFO [loop_until]: OK (rc = 0) 19:42:52 DEBUG --- stdout --- 19:42:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17186m 8650Mi am-767f6b67b5-qcqz2 17024m 9060Mi am-767f6b67b5-vbltz 18419m 8666Mi ds-cts-0 18873m 10232Mi ds-cts-1 17259m 10193Mi ds-cts-2 18845m 10243Mi ds-idrepo-0 7498m 3888Mi ds-idrepo-1 7502m 3882Mi ds-idrepo-2 7344m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 7m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8921m 733Mi 19:42:52 DEBUG --- stderr --- 19:42:52 DEBUG 19:42:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638573" 19:42:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638573" 19:42:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638573" 19:42:53 INFO [http_cmd]: http status code OK 19:42:53 DEBUG --- status code --- 19:42:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:53 DEBUG --- http response --- 19:42:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:53 DEBUG ----- output ----- 19:42:53 DEBUG { 19:42:53 DEBUG "status": "success", 19:42:53 DEBUG "data": { 19:42:53 DEBUG "resultType": "vector", 19:42:53 DEBUG "result": [ 19:42:53 DEBUG { 19:42:53 DEBUG "metric": { 19:42:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:53 DEBUG }, 19:42:53 DEBUG "value": [ 19:42:53 DEBUG 19:42:53 DEBUG 19:42:53 DEBUG 19:42:53 DEBUG [...] 19:42:53 DEBUG 19:42:53 DEBUG 19:42:53 DEBUG 19:42:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:53 DEBUG }, 19:42:53 DEBUG "value": [ 19:42:53 DEBUG 1713638573, 19:42:53 DEBUG "2671.9666666666667" 19:42:53 DEBUG ] 19:42:53 DEBUG } 19:42:53 DEBUG ] 19:42:53 DEBUG } 19:42:53 DEBUG } 19:42:53 INFO [http_cmd]: http status code OK 19:42:53 DEBUG --- status code --- 19:42:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:53 DEBUG --- http response --- 19:42:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:53 DEBUG ----- output ----- 19:42:53 DEBUG { 19:42:53 DEBUG "status": "success", 19:42:53 DEBUG "data": { 19:42:53 DEBUG "resultType": "vector", 19:42:53 DEBUG "result": [ 19:42:53 DEBUG { 19:42:53 DEBUG "metric": { 19:42:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:42:53 DEBUG }, 19:42:53 DEBUG "value": [ 19:42:53 DEBUG 19:42:53 INFO [http_cmd]: http status code OK 19:42:53 DEBUG 19:42:53 DEBUG --- status code --- 19:42:53 DEBUG 19:42:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:53 DEBUG [...] 19:42:53 DEBUG --- http response --- 19:42:53 DEBUG 19:42:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:53 DEBUG 19:42:53 DEBUG ----- output ----- 19:42:53 DEBUG 19:42:53 DEBUG { 19:42:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:42:53 DEBUG "status": "success", 19:42:53 DEBUG }, 19:42:53 DEBUG "data": { 19:42:53 DEBUG "value": [ 19:42:53 DEBUG "resultType": "vector", 19:42:53 DEBUG 1713638573, 19:42:53 DEBUG "result": [ 19:42:53 DEBUG "8.839306141593083" 19:42:53 DEBUG { 19:42:53 DEBUG ] 19:42:53 DEBUG "metric": { 19:42:53 DEBUG } 19:42:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:42:53 DEBUG ] 19:42:53 DEBUG }, 19:42:53 DEBUG } 19:42:53 DEBUG "value": [ 19:42:53 DEBUG } 19:42:53 DEBUG 19:42:53 DEBUG 19:42:53 DEBUG 19:42:53 DEBUG [...] 19:42:53 DEBUG 19:42:53 DEBUG 19:42:53 DEBUG 19:42:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:53 DEBUG }, 19:42:53 DEBUG "value": [ 19:42:53 DEBUG 1713638573, 19:42:53 DEBUG "2334.8177704941395" 19:42:53 DEBUG ] 19:42:53 DEBUG } 19:42:53 DEBUG ] 19:42:53 DEBUG } 19:42:53 DEBUG } 19:42:53 INFO 19:42:53 INFO [loop_until]: kubectl --namespace=xlou top node 19:42:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:42:53 INFO [loop_until]: OK (rc = 0) 19:42:53 DEBUG --- stdout --- 19:42:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6216Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7541m 25% 7637Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16801m 56% 11372Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19326m 64% 13475Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5184Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17232m 57% 11941Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14516m 48% 2948Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19447m 65% 10899Mi 9% gke-xlou-cdm-ds-0f601751-j488 8146m 27% 7183Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17339m 58% 15278Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 7858m 26% 7855Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19023m 63% 12930Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9074m 57% 6452Mi 11% 19:42:53 DEBUG --- stderr --- 19:42:53 DEBUG 19:42:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638573" 19:42:53 INFO [http_cmd]: http status code OK 19:42:53 DEBUG --- status code --- 19:42:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:53 DEBUG --- http response --- 19:42:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:53 DEBUG ----- output ----- 19:42:53 DEBUG { 19:42:53 DEBUG "status": "success", 19:42:53 DEBUG "data": { 19:42:53 DEBUG "resultType": "vector", 19:42:53 DEBUG "result": [ 19:42:53 DEBUG { 19:42:53 DEBUG "metric": { 19:42:53 DEBUG "instance": "10.142.0.107:9100" 19:42:53 DEBUG }, 19:42:53 DEBUG "value": [ 19:42:53 DEBUG 19:42:53 DEBUG 19:42:53 DEBUG 19:42:53 DEBUG [...] 19:42:53 DEBUG 19:42:53 DEBUG 19:42:53 DEBUG 19:42:53 DEBUG "instance": "10.142.0.43:9100" 19:42:53 DEBUG }, 19:42:53 DEBUG "value": [ 19:42:53 DEBUG 1713638573, 19:42:53 DEBUG "3577963643904" 19:42:53 DEBUG ] 19:42:53 DEBUG } 19:42:53 DEBUG ] 19:42:53 DEBUG } 19:42:53 DEBUG } 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638574" 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638574" 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 DEBUG --- status code --- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG { 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG "data": { 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "result": [ 19:42:54 DEBUG { 19:42:54 DEBUG "metric": { 19:42:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:54 DEBUG }, 19:42:54 DEBUG "value": [ 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG [...] 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:54 DEBUG }, 19:42:54 DEBUG "value": [ 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG "2672.6" 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638574" 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638574" 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 DEBUG --- status code --- 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638574" 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638574" 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 DEBUG --- status code --- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG --- status code --- 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 DEBUG --- status code --- 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 DEBUG --- status code --- 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 DEBUG --- status code --- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG --- status code --- 19:42:54 DEBUG --- status code --- 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 DEBUG --- status code --- 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG --- status code --- 19:42:54 DEBUG --- status code --- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG --- status code --- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG --- status code --- 19:42:54 DEBUG --- status code --- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG --- status code --- 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 INFO [http_cmd]: http status code OK 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG --- status code --- 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:42:54 DEBUG --- status code --- 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG { 19:42:54 DEBUG { 19:42:54 DEBUG { 19:42:54 DEBUG { 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG { 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG { 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG { 19:42:54 DEBUG { 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG { 19:42:54 DEBUG --- http response --- 19:42:54 DEBUG { 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG { 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:42:54 DEBUG { 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG "data": { 19:42:54 DEBUG "data": { 19:42:54 DEBUG "data": { 19:42:54 DEBUG "data": { 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG { 19:42:54 DEBUG { 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:42:54 DEBUG "data": { 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG "data": { 19:42:54 DEBUG "data": { 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG "data": { 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "data": { 19:42:54 DEBUG "data": { 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG "data": { 19:42:54 DEBUG { 19:42:54 DEBUG "data": { 19:42:54 DEBUG "data": { 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "result": [ 19:42:54 DEBUG { 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "result": [ 19:42:54 DEBUG { 19:42:54 DEBUG "result": [ 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "data": { 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG { 19:42:54 DEBUG "result": [ 19:42:54 DEBUG "metric": { 19:42:54 DEBUG ----- output ----- 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "result": [ 19:42:54 DEBUG "result": [ 19:42:54 DEBUG "metric": { 19:42:54 DEBUG { 19:42:54 DEBUG "result": [ 19:42:54 DEBUG "result": [ 19:42:54 DEBUG "result": [ 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "result": [ 19:42:54 DEBUG "data": { 19:42:54 DEBUG "result": [ 19:42:54 DEBUG "result": [ 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG { 19:42:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:54 DEBUG { 19:42:54 DEBUG "result": [ 19:42:54 DEBUG { 19:42:54 DEBUG { 19:42:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:42:54 DEBUG "metric": { 19:42:54 DEBUG { 19:42:54 DEBUG { 19:42:54 DEBUG { 19:42:54 DEBUG "result": [ 19:42:54 DEBUG { 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG { 19:42:54 DEBUG { 19:42:54 DEBUG "data": { 19:42:54 DEBUG "metric": { 19:42:54 DEBUG }, 19:42:54 DEBUG "status": "success", 19:42:54 DEBUG { 19:42:54 DEBUG "metric": { 19:42:54 DEBUG "metric": { 19:42:54 DEBUG }, 19:42:54 DEBUG "instance": "10.142.0.107:9100" 19:42:54 DEBUG "metric": { 19:42:54 DEBUG "metric": { 19:42:54 DEBUG "metric": { 19:42:54 DEBUG { 19:42:54 DEBUG "metric": { 19:42:54 DEBUG "result": [ 19:42:54 DEBUG "metric": { 19:42:54 DEBUG "metric": { 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "pod": "ds-idrepo-0" 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "data": { 19:42:54 DEBUG "metric": { 19:42:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:54 DEBUG "value": [ 19:42:54 DEBUG }, 19:42:54 DEBUG "instance": "10.142.0.107:9100" 19:42:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:42:54 DEBUG "instance": "10.142.0.107:9100" 19:42:54 DEBUG "metric": { 19:42:54 DEBUG "pod": "ds-idrepo-0" 19:42:54 DEBUG { 19:42:54 DEBUG "pod": "ds-cts-0" 19:42:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:42:54 DEBUG "result": [ 19:42:54 DEBUG }, 19:42:54 DEBUG 19:42:54 DEBUG "resultType": "vector", 19:42:54 DEBUG "pod": "ds-cts-0" 19:42:54 DEBUG }, 19:42:54 DEBUG }, 19:42:54 DEBUG 19:42:54 DEBUG "value": [ 19:42:54 DEBUG }, 19:42:54 DEBUG }, 19:42:54 DEBUG }, 19:42:54 DEBUG "pod": "ds-cts-0" 19:42:54 DEBUG }, 19:42:54 DEBUG "metric": { 19:42:54 DEBUG }, 19:42:54 DEBUG }, 19:42:54 DEBUG { 19:42:54 DEBUG "value": [ 19:42:54 DEBUG 19:42:54 DEBUG "result": [ 19:42:54 DEBUG }, 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "value": [ 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "value": [ 19:42:54 DEBUG }, 19:42:54 DEBUG "value": [ 19:42:54 DEBUG 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "metric": { 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG { 19:42:54 DEBUG "value": [ 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:42:54 DEBUG 19:42:54 DEBUG [...] 19:42:54 DEBUG "metric": { 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG [...] 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG }, 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG }, 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG "instance": "10.142.0.107:9100" 19:42:54 DEBUG 19:42:54 DEBUG [...] 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG "value": [ 19:42:54 DEBUG [...] 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG "value": [ 19:42:54 DEBUG [...] 19:42:54 DEBUG 19:42:54 DEBUG }, 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG [...] 19:42:54 DEBUG 19:42:54 DEBUG [...] 19:42:54 DEBUG [...] 19:42:54 DEBUG [...] 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG [...] 19:42:54 DEBUG [...] 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG "value": [ 19:42:54 DEBUG [...] 19:42:54 DEBUG 19:42:54 DEBUG [...] 19:42:54 DEBUG 19:42:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG [...] 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG }, 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG "pod": "ds-idrepo-2" 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG }, 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG "value": [ 19:42:54 DEBUG 19:42:54 DEBUG "instance": "10.142.0.43:9100" 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG [...] 19:42:54 DEBUG }, 19:42:54 DEBUG "value": [ 19:42:54 DEBUG [...] 19:42:54 DEBUG "pod": "ds-cts-2" 19:42:54 DEBUG 19:42:54 DEBUG "value": [ 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG }, 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "instance": "10.142.0.43:9100" 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG }, 19:42:54 DEBUG 19:42:54 DEBUG "instance": "10.142.0.43:9100" 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG 19:42:54 DEBUG }, 19:42:54 DEBUG 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG [...] 19:42:54 DEBUG 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG 19:42:54 DEBUG }, 19:42:54 DEBUG "865026048" 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:42:54 DEBUG }, 19:42:54 DEBUG "pod": "ds-cts-2" 19:42:54 DEBUG }, 19:42:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:54 DEBUG "0" 19:42:54 DEBUG 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "pod": "ds-cts-2" 19:42:54 DEBUG "0.154" 19:42:54 DEBUG 19:42:54 DEBUG "pod": "ds-cts-2" 19:42:54 DEBUG "2671.7" 19:42:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:54 DEBUG "value": [ 19:42:54 DEBUG ] 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG }, 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "value": [ 19:42:54 DEBUG }, 19:42:54 DEBUG ] 19:42:54 DEBUG 19:42:54 DEBUG 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG }, 19:42:54 DEBUG ] 19:42:54 DEBUG 19:42:54 DEBUG }, 19:42:54 DEBUG ] 19:42:54 DEBUG }, 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG } 19:42:54 DEBUG "0.0943333333333328" 19:42:54 DEBUG "value": [ 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG "value": [ 19:42:54 DEBUG } 19:42:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:54 DEBUG 19:42:54 DEBUG "0" 19:42:54 DEBUG "value": [ 19:42:54 DEBUG } 19:42:54 DEBUG "instance": "10.142.0.43:9100" 19:42:54 DEBUG "value": [ 19:42:54 DEBUG } 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "80874066.26666667" 19:42:54 DEBUG ] 19:42:54 DEBUG ] 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG "80878906.69999999" 19:42:54 DEBUG "0.0001" 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG ] 19:42:54 DEBUG }, 19:42:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:42:54 DEBUG ] 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG ] 19:42:54 DEBUG }, 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG ] 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG "0" 19:42:54 DEBUG ] 19:42:54 DEBUG ] 19:42:54 DEBUG "0" 19:42:54 DEBUG } 19:42:54 DEBUG "value": [ 19:42:54 DEBUG }, 19:42:54 DEBUG } 19:42:54 DEBUG "30" 19:42:54 DEBUG } 19:42:54 DEBUG "value": [ 19:42:54 DEBUG "0" 19:42:54 DEBUG } 19:42:54 DEBUG "0" 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG "value": [ 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG "0" 19:42:54 DEBUG 1713638574, 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG "0.26783333333334175" 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG ] 19:42:54 DEBUG "480233.5" 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG ] 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:42:54 DEBUG } 19:43:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638631" 19:43:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638631" 19:43:51 ERROR [http_cmd]: ERROR 19:43:51 INFO [http_cmd]: http status code OK 19:43:51 ERROR --- status code --- 19:43:51 DEBUG --- status code --- 19:43:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:43:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:51 ERROR --- http response --- 19:43:51 DEBUG --- http response --- 19:43:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:43:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:43:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638632" 19:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638632" 19:43:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638632" 19:43:52 INFO [http_cmd]: http status code OK 19:43:52 INFO [http_cmd]: http status code OK 19:43:52 INFO [http_cmd]: http status code OK 19:43:52 DEBUG --- status code --- 19:43:52 DEBUG --- status code --- 19:43:52 DEBUG --- status code --- 19:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:52 DEBUG --- http response --- 19:43:52 DEBUG --- http response --- 19:43:52 DEBUG --- http response --- 19:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:52 DEBUG ----- output ----- 19:43:52 DEBUG ----- output ----- 19:43:52 DEBUG ----- output ----- 19:43:52 DEBUG { 19:43:52 DEBUG { 19:43:52 DEBUG { 19:43:52 DEBUG "status": "success", 19:43:52 DEBUG "status": "success", 19:43:52 DEBUG "status": "success", 19:43:52 DEBUG "data": { 19:43:52 DEBUG "data": { 19:43:52 DEBUG "data": { 19:43:52 DEBUG "resultType": "vector", 19:43:52 DEBUG "resultType": "vector", 19:43:52 DEBUG "resultType": "vector", 19:43:52 DEBUG "result": [ 19:43:52 DEBUG "result": [ 19:43:52 DEBUG "result": [ 19:43:52 DEBUG { 19:43:52 DEBUG { 19:43:52 DEBUG { 19:43:52 DEBUG "metric": { 19:43:52 DEBUG "metric": { 19:43:52 DEBUG "metric": { 19:43:52 DEBUG "pod": "ds-idrepo-0" 19:43:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:52 DEBUG "pod": "ds-idrepo-0" 19:43:52 DEBUG }, 19:43:52 DEBUG }, 19:43:52 DEBUG }, 19:43:52 DEBUG "value": [ 19:43:52 DEBUG "value": [ 19:43:52 DEBUG "value": [ 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG [...] 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG [...] 19:43:52 DEBUG [...] 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG "pod": "ds-idrepo-2" 19:43:52 DEBUG 19:43:52 DEBUG 19:43:52 DEBUG }, 19:43:52 DEBUG "pod": "ds-idrepo-2" 19:43:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:52 DEBUG "value": [ 19:43:52 DEBUG }, 19:43:52 DEBUG }, 19:43:52 DEBUG 1713638632, 19:43:52 DEBUG "value": [ 19:43:52 DEBUG "value": [ 19:43:52 DEBUG "0" 19:43:52 DEBUG 1713638632, 19:43:52 DEBUG 1713638632, 19:43:52 DEBUG ] 19:43:52 DEBUG "0" 19:43:52 DEBUG "2661.433333333334" 19:43:52 DEBUG } 19:43:52 DEBUG ] 19:43:52 DEBUG ] 19:43:52 DEBUG ] 19:43:52 DEBUG } 19:43:52 DEBUG } 19:43:52 DEBUG } 19:43:52 DEBUG } 19:43:52 DEBUG ] 19:43:52 DEBUG ] 19:43:52 DEBUG } 19:43:52 DEBUG } 19:43:52 DEBUG } 19:43:52 DEBUG } 19:43:52 INFO 19:43:52 INFO [loop_until]: kubectl --namespace=xlou top pods 19:43:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:43:52 INFO [loop_until]: OK (rc = 0) 19:43:52 DEBUG --- stdout --- 19:43:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17209m 8648Mi am-767f6b67b5-qcqz2 16874m 9060Mi am-767f6b67b5-vbltz 18692m 8668Mi ds-cts-0 18602m 10255Mi ds-cts-1 17189m 10187Mi ds-cts-2 19204m 10237Mi ds-idrepo-0 7307m 3888Mi ds-idrepo-1 7394m 3882Mi ds-idrepo-2 7452m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 14m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8779m 735Mi 19:43:52 DEBUG --- stderr --- 19:43:52 DEBUG 19:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638633" 19:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638633" 19:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638633" 19:43:53 INFO [http_cmd]: http status code OK 19:43:53 DEBUG --- status code --- 19:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:53 DEBUG --- http response --- 19:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:53 DEBUG ----- output ----- 19:43:53 DEBUG { 19:43:53 DEBUG "status": "success", 19:43:53 DEBUG "data": { 19:43:53 DEBUG "resultType": "vector", 19:43:53 DEBUG "result": [ 19:43:53 DEBUG { 19:43:53 DEBUG "metric": { 19:43:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:53 DEBUG }, 19:43:53 DEBUG "value": [ 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG [...] 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:53 DEBUG }, 19:43:53 DEBUG "value": [ 19:43:53 DEBUG 1713638633, 19:43:53 DEBUG "2670.1666666666665" 19:43:53 DEBUG ] 19:43:53 DEBUG } 19:43:53 DEBUG ] 19:43:53 DEBUG } 19:43:53 DEBUG } 19:43:53 INFO [http_cmd]: http status code OK 19:43:53 DEBUG --- status code --- 19:43:53 INFO [http_cmd]: http status code OK 19:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:53 DEBUG --- status code --- 19:43:53 DEBUG --- http response --- 19:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:43:53 DEBUG --- http response --- 19:43:53 DEBUG ----- output ----- 19:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:53 DEBUG { 19:43:53 DEBUG ----- output ----- 19:43:53 DEBUG "status": "success", 19:43:53 DEBUG { 19:43:53 DEBUG "data": { 19:43:53 DEBUG "status": "success", 19:43:53 DEBUG "resultType": "vector", 19:43:53 DEBUG "data": { 19:43:53 DEBUG "result": [ 19:43:53 DEBUG "resultType": "vector", 19:43:53 DEBUG { 19:43:53 DEBUG "result": [ 19:43:53 DEBUG "metric": { 19:43:53 DEBUG { 19:43:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:43:53 DEBUG "metric": { 19:43:53 DEBUG }, 19:43:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:43:53 DEBUG "value": [ 19:43:53 DEBUG }, 19:43:53 DEBUG 19:43:53 DEBUG "value": [ 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG [...] 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG [...] 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:43:53 DEBUG }, 19:43:53 DEBUG "value": [ 19:43:53 DEBUG 1713638633, 19:43:53 DEBUG "8.862610680276756" 19:43:53 DEBUG ] 19:43:53 DEBUG } 19:43:53 DEBUG ] 19:43:53 DEBUG "pod": "ds-cts-0" 19:43:53 DEBUG } 19:43:53 DEBUG }, 19:43:53 DEBUG } 19:43:53 DEBUG "value": [ 19:43:53 DEBUG 1713638633, 19:43:53 DEBUG "67416528.18877898" 19:43:53 DEBUG ] 19:43:53 DEBUG } 19:43:53 DEBUG ] 19:43:53 DEBUG } 19:43:53 DEBUG } 19:43:53 INFO 19:43:53 INFO [loop_until]: kubectl --namespace=xlou top node 19:43:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:43:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638633" 19:43:53 INFO [http_cmd]: http status code OK 19:43:53 DEBUG --- status code --- 19:43:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:53 DEBUG --- http response --- 19:43:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:53 DEBUG ----- output ----- 19:43:53 DEBUG { 19:43:53 DEBUG "status": "success", 19:43:53 DEBUG "data": { 19:43:53 DEBUG "resultType": "vector", 19:43:53 DEBUG "result": [ 19:43:53 DEBUG { 19:43:53 DEBUG "metric": { 19:43:53 DEBUG "instance": "10.142.0.107:9100" 19:43:53 DEBUG }, 19:43:53 DEBUG "value": [ 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG [...] 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG 19:43:53 DEBUG "instance": "10.142.0.43:9100" 19:43:53 DEBUG }, 19:43:53 DEBUG "value": [ 19:43:53 DEBUG 1713638633, 19:43:53 DEBUG "3581945967616" 19:43:53 DEBUG ] 19:43:53 DEBUG } 19:43:53 DEBUG ] 19:43:53 DEBUG } 19:43:53 DEBUG } 19:43:53 INFO [loop_until]: OK (rc = 0) 19:43:53 DEBUG --- stdout --- 19:43:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6222Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7569m 25% 7635Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16983m 56% 11370Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3358Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18297m 61% 13480Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5183Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17257m 57% 11948Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14557m 48% 2961Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19515m 65% 10892Mi 9% gke-xlou-cdm-ds-0f601751-j488 8175m 27% 7185Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17254m 57% 15262Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8049m 26% 7857Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19264m 64% 12930Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9086m 57% 6369Mi 10% 19:43:53 DEBUG --- stderr --- 19:43:53 DEBUG 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG { 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "data": { 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "result": [ 19:43:54 DEBUG { 19:43:54 DEBUG "metric": { 19:43:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:54 DEBUG }, 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG [...] 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:54 DEBUG }, 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG "2670.733333333333" 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG { 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "data": { 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "result": [ 19:43:54 DEBUG { 19:43:54 DEBUG "metric": { 19:43:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:54 DEBUG }, 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG [...] 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:54 DEBUG }, 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG "2671.6" 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638634" 19:43:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638634" 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 DEBUG --- status code --- 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 DEBUG --- status code --- 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG --- status code --- 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG --- http response --- 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 INFO [http_cmd]: http status code OK 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG --- status code --- 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG { 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG { 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG { 19:43:54 DEBUG { 19:43:54 DEBUG { 19:43:54 DEBUG { 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG --- http response --- 19:43:54 DEBUG { 19:43:54 DEBUG { 19:43:54 DEBUG { 19:43:54 DEBUG { 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG "data": { 19:43:54 DEBUG { 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG { 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "data": { 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "data": { 19:43:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:43:54 DEBUG { 19:43:54 DEBUG { 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "data": { 19:43:54 DEBUG "data": { 19:43:54 DEBUG "data": { 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "data": { 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "result": [ 19:43:54 DEBUG "data": { 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG ----- output ----- 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "result": [ 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "result": [ 19:43:54 DEBUG { 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "data": { 19:43:54 DEBUG { 19:43:54 DEBUG "data": { 19:43:54 DEBUG { 19:43:54 DEBUG "metric": { 19:43:54 DEBUG "data": { 19:43:54 DEBUG "result": [ 19:43:54 DEBUG { 19:43:54 DEBUG "data": { 19:43:54 DEBUG "result": [ 19:43:54 DEBUG { 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "result": [ 19:43:54 DEBUG "metric": { 19:43:54 DEBUG "data": { 19:43:54 DEBUG "result": [ 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "pod": "ds-cts-0" 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "metric": { 19:43:54 DEBUG { 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG { 19:43:54 DEBUG "metric": { 19:43:54 DEBUG "data": { 19:43:54 DEBUG { 19:43:54 DEBUG "metric": { 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG { 19:43:54 DEBUG "result": [ 19:43:54 DEBUG "data": { 19:43:54 DEBUG "result": [ 19:43:54 DEBUG }, 19:43:54 DEBUG "result": [ 19:43:54 DEBUG "result": [ 19:43:54 DEBUG "pod": "ds-idrepo-0" 19:43:54 DEBUG "status": "success", 19:43:54 DEBUG "data": { 19:43:54 DEBUG "metric": { 19:43:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:54 DEBUG "result": [ 19:43:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:43:54 DEBUG "metric": { 19:43:54 DEBUG { 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG { 19:43:54 DEBUG "value": [ 19:43:54 DEBUG { 19:43:54 DEBUG "metric": { 19:43:54 DEBUG }, 19:43:54 DEBUG "result": [ 19:43:54 DEBUG "resultType": "vector", 19:43:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:54 DEBUG }, 19:43:54 DEBUG "result": [ 19:43:54 DEBUG { 19:43:54 DEBUG { 19:43:54 DEBUG }, 19:43:54 DEBUG "instance": "10.142.0.107:9100" 19:43:54 DEBUG "metric": { 19:43:54 DEBUG "result": [ 19:43:54 DEBUG "metric": { 19:43:54 DEBUG 19:43:54 DEBUG "instance": "10.142.0.107:9100" 19:43:54 DEBUG { 19:43:54 DEBUG "value": [ 19:43:54 DEBUG { 19:43:54 DEBUG "result": [ 19:43:54 DEBUG }, 19:43:54 DEBUG "value": [ 19:43:54 DEBUG "metric": { 19:43:54 DEBUG }, 19:43:54 DEBUG "metric": { 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 19:43:54 DEBUG "pod": "ds-cts-0" 19:43:54 DEBUG { 19:43:54 DEBUG "pod": "ds-idrepo-0" 19:43:54 DEBUG 19:43:54 DEBUG }, 19:43:54 DEBUG "metric": { 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "metric": { 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 19:43:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:43:54 DEBUG }, 19:43:54 DEBUG "instance": "10.142.0.107:9100" 19:43:54 DEBUG }, 19:43:54 DEBUG }, 19:43:54 DEBUG }, 19:43:54 DEBUG "metric": { 19:43:54 DEBUG }, 19:43:54 DEBUG 19:43:54 DEBUG "value": [ 19:43:54 DEBUG "pod": "ds-cts-0" 19:43:54 DEBUG { 19:43:54 DEBUG 19:43:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "value": [ 19:43:54 DEBUG "value": [ 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "value": [ 19:43:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:43:54 DEBUG }, 19:43:54 DEBUG [...] 19:43:54 DEBUG 19:43:54 DEBUG }, 19:43:54 DEBUG "metric": { 19:43:54 DEBUG [...] 19:43:54 DEBUG }, 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "value": [ 19:43:54 DEBUG [...] 19:43:54 DEBUG 19:43:54 DEBUG "value": [ 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "value": [ 19:43:54 DEBUG "instance": "10.142.0.107:9100" 19:43:54 DEBUG 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 19:43:54 DEBUG [...] 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG }, 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG [...] 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "pod": "ds-cts-2" 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG [...] 19:43:54 DEBUG [...] 19:43:54 DEBUG [...] 19:43:54 DEBUG 19:43:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG }, 19:43:54 DEBUG [...] 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "pod": "ds-idrepo-2" 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG [...] 19:43:54 DEBUG }, 19:43:54 DEBUG [...] 19:43:54 DEBUG [...] 19:43:54 DEBUG 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 19:43:54 DEBUG [...] 19:43:54 DEBUG 19:43:54 DEBUG }, 19:43:54 DEBUG [...] 19:43:54 DEBUG 19:43:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 19:43:54 DEBUG [...] 19:43:54 DEBUG 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG "34.766666666666666" 19:43:54 DEBUG ] 19:43:54 DEBUG 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 19:43:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:54 DEBUG }, 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG } 19:43:54 DEBUG 19:43:54 DEBUG [...] 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG 19:43:54 DEBUG }, 19:43:54 DEBUG "value": [ 19:43:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:43:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:54 DEBUG "instance": "10.142.0.43:9100" 19:43:54 DEBUG }, 19:43:54 DEBUG "867917824" 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "pod": "ds-cts-2" 19:43:54 DEBUG 19:43:54 DEBUG ] 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG "0" 19:43:54 DEBUG 19:43:54 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG }, 19:43:54 DEBUG }, 19:43:54 DEBUG 19:43:54 DEBUG "value": [ 19:43:54 DEBUG ] 19:43:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:43:54 DEBUG 19:43:54 DEBUG }, 19:43:54 DEBUG "instance": "10.142.0.43:9100" 19:43:54 DEBUG } 19:43:54 DEBUG 19:43:54 DEBUG 19:43:54 DEBUG ] 19:43:54 DEBUG "value": [ 19:43:54 DEBUG }, 19:43:54 DEBUG "0" 19:43:54 DEBUG "value": [ 19:43:54 DEBUG "value": [ 19:43:54 DEBUG "instance": "10.142.0.43:9100" 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG } 19:43:54 DEBUG }, 19:43:54 DEBUG "pod": "ds-cts-2" 19:43:54 DEBUG "value": [ 19:43:54 DEBUG }, 19:43:54 DEBUG } 19:43:54 DEBUG "pod": "ds-cts-2" 19:43:54 DEBUG 19:43:54 DEBUG } 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG "value": [ 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG }, 19:43:54 DEBUG "80918455.03333333" 19:43:54 DEBUG ] 19:43:54 DEBUG "value": [ 19:43:54 DEBUG }, 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG "value": [ 19:43:54 DEBUG }, 19:43:54 DEBUG "instance": "10.142.0.43:9100" 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG ] 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG "0.154" 19:43:54 DEBUG "value": [ 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG "value": [ 19:43:54 DEBUG "0" 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG "value": [ 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG "0" 19:43:54 DEBUG } 19:43:54 DEBUG "0" 19:43:54 DEBUG } 19:43:54 DEBUG "0" 19:43:54 DEBUG ] 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG } 19:43:54 DEBUG ] 19:43:54 DEBUG "480056" 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG ] 19:43:54 DEBUG "80913030.3" 19:43:54 DEBUG ] 19:43:54 DEBUG "0.0000666666666666667" 19:43:54 DEBUG ] 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG "0.060333333333337194" 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG }, 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG ] 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG "0" 19:43:54 DEBUG ] 19:43:54 DEBUG ] 19:43:54 DEBUG ] 19:43:54 DEBUG "value": [ 19:43:54 DEBUG } 19:43:54 DEBUG ] 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG 1713638634, 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG ] 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG "0.16036666666661575" 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG ] 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG } 19:43:54 DEBUG ] 19:43:54 DEBUG } 19:43:54 DEBUG } 19:44:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638691" 19:44:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638691" 19:44:51 INFO [http_cmd]: http status code OK 19:44:51 ERROR [http_cmd]: ERROR 19:44:51 DEBUG --- status code --- 19:44:51 ERROR --- status code --- 19:44:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:44:51 DEBUG --- http response --- 19:44:51 ERROR --- http response --- 19:44:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:44:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:44:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638692" 19:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638692" 19:44:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638692" 19:44:52 INFO [http_cmd]: http status code OK 19:44:52 INFO [http_cmd]: http status code OK 19:44:52 DEBUG --- status code --- 19:44:52 DEBUG --- status code --- 19:44:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:52 DEBUG --- http response --- 19:44:52 DEBUG --- http response --- 19:44:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:52 DEBUG ----- output ----- 19:44:52 DEBUG { 19:44:52 DEBUG "status": "success", 19:44:52 DEBUG "data": { 19:44:52 DEBUG "resultType": "vector", 19:44:52 DEBUG "result": [ 19:44:52 DEBUG { 19:44:52 DEBUG "metric": { 19:44:52 DEBUG "pod": "ds-idrepo-0" 19:44:52 DEBUG }, 19:44:52 DEBUG "value": [ 19:44:52 DEBUG 19:44:52 DEBUG 19:44:52 DEBUG 19:44:52 DEBUG [...] 19:44:52 DEBUG 19:44:52 DEBUG 19:44:52 DEBUG 19:44:52 DEBUG "pod": "ds-idrepo-2" 19:44:52 DEBUG }, 19:44:52 DEBUG "value": [ 19:44:52 DEBUG 1713638692, 19:44:52 DEBUG "0" 19:44:52 DEBUG ] 19:44:52 DEBUG } 19:44:52 DEBUG ] 19:44:52 DEBUG } 19:44:52 DEBUG } 19:44:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:52 DEBUG ----- output ----- 19:44:52 DEBUG { 19:44:52 DEBUG "status": "success", 19:44:52 DEBUG "data": { 19:44:52 DEBUG "resultType": "vector", 19:44:52 DEBUG "result": [ 19:44:52 DEBUG { 19:44:52 DEBUG "metric": { 19:44:52 DEBUG "pod": "ds-idrepo-0" 19:44:52 DEBUG }, 19:44:52 DEBUG "value": [ 19:44:52 DEBUG 19:44:52 DEBUG 19:44:52 DEBUG 19:44:52 DEBUG [...] 19:44:52 DEBUG 19:44:52 DEBUG 19:44:52 DEBUG 19:44:52 DEBUG "pod": "ds-idrepo-2" 19:44:52 DEBUG }, 19:44:52 DEBUG "value": [ 19:44:52 DEBUG 1713638692, 19:44:52 DEBUG "0" 19:44:52 DEBUG ] 19:44:52 DEBUG } 19:44:52 DEBUG ] 19:44:52 DEBUG } 19:44:52 DEBUG } 19:44:52 INFO [http_cmd]: http status code OK 19:44:52 DEBUG --- status code --- 19:44:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:52 DEBUG --- http response --- 19:44:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:52 DEBUG ----- output ----- 19:44:52 DEBUG { 19:44:52 DEBUG "status": "success", 19:44:52 DEBUG "data": { 19:44:52 DEBUG "resultType": "vector", 19:44:52 DEBUG "result": [ 19:44:52 DEBUG { 19:44:52 DEBUG "metric": { 19:44:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:52 DEBUG }, 19:44:52 DEBUG "value": [ 19:44:52 DEBUG 19:44:52 DEBUG 19:44:52 DEBUG 19:44:52 DEBUG [...] 19:44:52 DEBUG 19:44:52 DEBUG 19:44:52 DEBUG 19:44:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:52 DEBUG }, 19:44:52 DEBUG "value": [ 19:44:52 DEBUG 1713638692, 19:44:52 DEBUG "2664.4000000000005" 19:44:52 DEBUG ] 19:44:52 DEBUG } 19:44:52 DEBUG ] 19:44:52 DEBUG } 19:44:52 DEBUG } 19:44:52 INFO 19:44:52 INFO [loop_until]: kubectl --namespace=xlou top pods 19:44:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:44:53 INFO [loop_until]: OK (rc = 0) 19:44:53 DEBUG --- stdout --- 19:44:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16849m 8650Mi am-767f6b67b5-qcqz2 17131m 9061Mi am-767f6b67b5-vbltz 18081m 8667Mi ds-cts-0 18334m 10258Mi ds-cts-1 17578m 10199Mi ds-cts-2 19569m 10247Mi ds-idrepo-0 7459m 3888Mi ds-idrepo-1 7587m 3882Mi ds-idrepo-2 7392m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8682m 738Mi 19:44:53 DEBUG --- stderr --- 19:44:53 DEBUG 19:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638693" 19:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638693" 19:44:53 INFO [http_cmd]: http status code OK 19:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638693" 19:44:53 DEBUG --- status code --- 19:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:53 DEBUG --- http response --- 19:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:53 DEBUG ----- output ----- 19:44:53 DEBUG { 19:44:53 DEBUG "status": "success", 19:44:53 DEBUG "data": { 19:44:53 DEBUG "resultType": "vector", 19:44:53 DEBUG "result": [ 19:44:53 DEBUG { 19:44:53 DEBUG "metric": { 19:44:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:53 DEBUG }, 19:44:53 DEBUG "value": [ 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG [...] 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:53 DEBUG }, 19:44:53 DEBUG "value": [ 19:44:53 DEBUG 1713638693, 19:44:53 DEBUG "2665.866666666667" 19:44:53 DEBUG ] 19:44:53 DEBUG } 19:44:53 DEBUG ] 19:44:53 DEBUG } 19:44:53 DEBUG } 19:44:53 INFO [http_cmd]: http status code OK 19:44:53 DEBUG --- status code --- 19:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:53 DEBUG --- http response --- 19:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:44:53 DEBUG ----- output ----- 19:44:53 DEBUG { 19:44:53 DEBUG "status": "success", 19:44:53 DEBUG "data": { 19:44:53 DEBUG "resultType": "vector", 19:44:53 DEBUG "result": [ 19:44:53 DEBUG { 19:44:53 DEBUG "metric": { 19:44:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:44:53 DEBUG }, 19:44:53 DEBUG "value": [ 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG [...] 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG "pod": "ds-idrepo-0" 19:44:53 DEBUG }, 19:44:53 DEBUG "value": [ 19:44:53 DEBUG 1713638693, 19:44:53 DEBUG "5314.348448052164" 19:44:53 DEBUG ] 19:44:53 DEBUG } 19:44:53 DEBUG ] 19:44:53 DEBUG } 19:44:53 DEBUG } 19:44:53 INFO [http_cmd]: http status code OK 19:44:53 DEBUG --- status code --- 19:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:53 DEBUG --- http response --- 19:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:53 DEBUG ----- output ----- 19:44:53 DEBUG { 19:44:53 DEBUG "status": "success", 19:44:53 DEBUG "data": { 19:44:53 DEBUG "resultType": "vector", 19:44:53 DEBUG "result": [ 19:44:53 DEBUG { 19:44:53 DEBUG "metric": { 19:44:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:44:53 DEBUG }, 19:44:53 DEBUG "value": [ 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG [...] 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:44:53 DEBUG }, 19:44:53 DEBUG "value": [ 19:44:53 DEBUG 1713638693, 19:44:53 DEBUG "8.860934397375159" 19:44:53 DEBUG ] 19:44:53 DEBUG } 19:44:53 DEBUG ] 19:44:53 DEBUG } 19:44:53 DEBUG } 19:44:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638693" 19:44:53 INFO [http_cmd]: http status code OK 19:44:53 DEBUG --- status code --- 19:44:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:53 DEBUG --- http response --- 19:44:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:53 DEBUG ----- output ----- 19:44:53 DEBUG { 19:44:53 DEBUG "status": "success", 19:44:53 DEBUG "data": { 19:44:53 DEBUG "resultType": "vector", 19:44:53 DEBUG "result": [ 19:44:53 DEBUG { 19:44:53 DEBUG "metric": { 19:44:53 DEBUG "instance": "10.142.0.107:9100" 19:44:53 DEBUG }, 19:44:53 DEBUG "value": [ 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG [...] 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG 19:44:53 DEBUG "instance": "10.142.0.43:9100" 19:44:53 DEBUG }, 19:44:53 DEBUG "value": [ 19:44:53 DEBUG 1713638693, 19:44:53 DEBUG "3586290119680" 19:44:53 DEBUG ] 19:44:53 DEBUG } 19:44:53 DEBUG ] 19:44:53 DEBUG } 19:44:53 DEBUG } 19:44:53 INFO 19:44:53 INFO [loop_until]: kubectl --namespace=xlou top node 19:44:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:44:53 INFO [loop_until]: OK (rc = 0) 19:44:53 DEBUG --- stdout --- 19:44:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6209Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7564m 25% 7647Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16995m 56% 11363Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 89m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18842m 63% 13480Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 141m 0% 5212Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17269m 57% 11939Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14510m 48% 2963Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18835m 63% 10897Mi 9% gke-xlou-cdm-ds-0f601751-j488 8188m 27% 7177Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16646m 55% 15282Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8042m 26% 7852Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19421m 65% 12936Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9069m 57% 6370Mi 10% 19:44:53 DEBUG --- stderr --- 19:44:53 DEBUG 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG { 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG "data": { 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "result": [ 19:44:54 DEBUG { 19:44:54 DEBUG "metric": { 19:44:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:54 DEBUG }, 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:54 DEBUG }, 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG "2663.6666666666665" 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG { 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG "data": { 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "result": [ 19:44:54 DEBUG { 19:44:54 DEBUG "metric": { 19:44:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:54 DEBUG }, 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:54 DEBUG }, 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG "2663.766666666667" 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638694" 19:44:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638694" 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 DEBUG --- status code --- 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 DEBUG --- status code --- 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 DEBUG --- status code --- 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG --- status code --- 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG --- status code --- 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG --- http response --- 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 INFO [http_cmd]: http status code OK 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:54 DEBUG --- status code --- 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG { 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:54 DEBUG { 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG { 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG { 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG { 19:44:54 DEBUG { 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG "data": { 19:44:54 DEBUG { 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG --- http response --- 19:44:54 DEBUG "data": { 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:54 DEBUG { 19:44:54 DEBUG { 19:44:54 DEBUG { 19:44:54 DEBUG "data": { 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG { 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG { 19:44:54 DEBUG { 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG { 19:44:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "data": { 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG "data": { 19:44:54 DEBUG "data": { 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "data": { 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG ----- output ----- 19:44:54 DEBUG "result": [ 19:44:54 DEBUG { 19:44:54 DEBUG "data": { 19:44:54 DEBUG "data": { 19:44:54 DEBUG "data": { 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "data": { 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "data": { 19:44:54 DEBUG "data": { 19:44:54 DEBUG { 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "data": { 19:44:54 DEBUG { 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG "status": "success", 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG { 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "metric": { 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG { 19:44:54 DEBUG "data": { 19:44:54 DEBUG "data": { 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "metric": { 19:44:54 DEBUG { 19:44:54 DEBUG "result": [ 19:44:54 DEBUG { 19:44:54 DEBUG { 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "instance": "10.142.0.107:9100" 19:44:54 DEBUG { 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "metric": { 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG "result": [ 19:44:54 DEBUG { 19:44:54 DEBUG { 19:44:54 DEBUG { 19:44:54 DEBUG "pod": "ds-cts-0" 19:44:54 DEBUG "metric": { 19:44:54 DEBUG { 19:44:54 DEBUG "metric": { 19:44:54 DEBUG "metric": { 19:44:54 DEBUG { 19:44:54 DEBUG { 19:44:54 DEBUG }, 19:44:54 DEBUG "metric": { 19:44:54 DEBUG { 19:44:54 DEBUG "pod": "ds-idrepo-0" 19:44:54 DEBUG "resultType": "vector", 19:44:54 DEBUG { 19:44:54 DEBUG "metric": { 19:44:54 DEBUG "metric": { 19:44:54 DEBUG "metric": { 19:44:54 DEBUG }, 19:44:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:44:54 DEBUG "metric": { 19:44:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:44:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:54 DEBUG "metric": { 19:44:54 DEBUG "metric": { 19:44:54 DEBUG "value": [ 19:44:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:44:54 DEBUG "metric": { 19:44:54 DEBUG }, 19:44:54 DEBUG "result": [ 19:44:54 DEBUG "metric": { 19:44:54 DEBUG "instance": "10.142.0.107:9100" 19:44:54 DEBUG "pod": "ds-cts-0" 19:44:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:54 DEBUG "value": [ 19:44:54 DEBUG }, 19:44:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:44:54 DEBUG }, 19:44:54 DEBUG "value": [ 19:44:54 DEBUG }, 19:44:54 DEBUG "pod": "ds-cts-0" 19:44:54 DEBUG 19:44:54 DEBUG }, 19:44:54 DEBUG "pod": "ds-idrepo-0" 19:44:54 DEBUG "value": [ 19:44:54 DEBUG { 19:44:54 DEBUG "instance": "10.142.0.107:9100" 19:44:54 DEBUG }, 19:44:54 DEBUG }, 19:44:54 DEBUG }, 19:44:54 DEBUG 19:44:54 DEBUG "value": [ 19:44:54 DEBUG }, 19:44:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:44:54 DEBUG 19:44:54 DEBUG "value": [ 19:44:54 DEBUG }, 19:44:54 DEBUG 19:44:54 DEBUG "value": [ 19:44:54 DEBUG }, 19:44:54 DEBUG 19:44:54 DEBUG "metric": { 19:44:54 DEBUG }, 19:44:54 DEBUG "value": [ 19:44:54 DEBUG "value": [ 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG "value": [ 19:44:54 DEBUG }, 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 19:44:54 DEBUG "instance": "10.142.0.107:9100" 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG }, 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG [...] 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG "instance": "10.142.0.43:9100" 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG [...] 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG "pod": "ds-cts-2" 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG }, 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG "pod": "ds-idrepo-2" 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG }, 19:44:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:44:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:54 DEBUG 19:44:54 DEBUG "value": [ 19:44:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:44:54 DEBUG 19:44:54 DEBUG }, 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG 19:44:54 DEBUG "pod": "ds-cts-2" 19:44:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:54 DEBUG "value": [ 19:44:54 DEBUG }, 19:44:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:44:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:44:54 DEBUG }, 19:44:54 DEBUG }, 19:44:54 DEBUG "pod": "ds-cts-2" 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG }, 19:44:54 DEBUG "pod": "ds-cts-2" 19:44:54 DEBUG "value": [ 19:44:54 DEBUG "instance": "10.142.0.43:9100" 19:44:54 DEBUG 19:44:54 DEBUG "instance": "10.142.0.43:9100" 19:44:54 DEBUG }, 19:44:54 DEBUG }, 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG "value": [ 19:44:54 DEBUG }, 19:44:54 DEBUG "value": [ 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG }, 19:44:54 DEBUG "80528900.93333334" 19:44:54 DEBUG "value": [ 19:44:54 DEBUG }, 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG }, 19:44:54 DEBUG "instance": "10.142.0.43:9100" 19:44:54 DEBUG }, 19:44:54 DEBUG "value": [ 19:44:54 DEBUG "value": [ 19:44:54 DEBUG "29.933333333333334" 19:44:54 DEBUG ] 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG }, 19:44:54 DEBUG "0" 19:44:54 DEBUG "value": [ 19:44:54 DEBUG "value": [ 19:44:54 DEBUG ] 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG "5262.6676302898895" 19:44:54 DEBUG "0" 19:44:54 DEBUG "value": [ 19:44:54 DEBUG }, 19:44:54 DEBUG "value": [ 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG } 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG "8.827204627760372" 19:44:54 DEBUG "value": [ 19:44:54 DEBUG ] 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG ] 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG "value": [ 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG "0" 19:44:54 DEBUG ] 19:44:54 DEBUG "870764544" 19:44:54 DEBUG ] 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG } 19:44:54 DEBUG "0.154" 19:44:54 DEBUG "0" 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG "0.2607333333333562" 19:44:54 DEBUG "0.0956666666666706" 19:44:54 DEBUG 1713638694, 19:44:54 DEBUG "0.00039999999999999996" 19:44:54 DEBUG "0" 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG "480093" 19:44:54 DEBUG ] 19:44:54 DEBUG ] 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG "80524253.9" 19:44:54 DEBUG ] 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG ] 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG ] 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:44:54 DEBUG } 19:45:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638751" 19:45:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638751" 19:45:51 ERROR [http_cmd]: ERROR 19:45:51 ERROR --- status code --- 19:45:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:45:51 ERROR --- http response --- 19:45:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:45:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:45:51 INFO [http_cmd]: http status code OK 19:45:51 DEBUG --- status code --- 19:45:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:51 DEBUG --- http response --- 19:45:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:45:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638752" 19:45:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638752" 19:45:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638752" 19:45:52 INFO [http_cmd]: http status code OK 19:45:52 DEBUG --- status code --- 19:45:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:52 INFO [http_cmd]: http status code OK 19:45:52 DEBUG --- http response --- 19:45:52 DEBUG --- status code --- 19:45:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:52 DEBUG --- http response --- 19:45:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:52 DEBUG ----- output ----- 19:45:52 DEBUG ----- output ----- 19:45:52 DEBUG { 19:45:52 DEBUG { 19:45:52 DEBUG "status": "success", 19:45:52 DEBUG "status": "success", 19:45:52 DEBUG "data": { 19:45:52 DEBUG "data": { 19:45:52 DEBUG "resultType": "vector", 19:45:52 DEBUG "resultType": "vector", 19:45:52 DEBUG "result": [ 19:45:52 DEBUG "result": [ 19:45:52 DEBUG { 19:45:52 DEBUG { 19:45:52 DEBUG "metric": { 19:45:52 DEBUG "metric": { 19:45:52 DEBUG "pod": "ds-idrepo-0" 19:45:52 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:52 DEBUG }, 19:45:52 DEBUG }, 19:45:52 DEBUG "value": [ 19:45:52 DEBUG "value": [ 19:45:52 DEBUG 19:45:52 DEBUG 19:45:52 INFO [http_cmd]: http status code OK 19:45:52 DEBUG 19:45:52 DEBUG 19:45:52 DEBUG --- status code --- 19:45:52 DEBUG 19:45:52 DEBUG 19:45:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:52 DEBUG [...] 19:45:52 DEBUG [...] 19:45:52 DEBUG --- http response --- 19:45:52 DEBUG 19:45:52 DEBUG 19:45:52 DEBUG 19:45:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:52 DEBUG 19:45:52 DEBUG 19:45:52 DEBUG ----- output ----- 19:45:52 DEBUG 19:45:52 DEBUG "pod": "ds-idrepo-2" 19:45:52 DEBUG { 19:45:52 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:52 DEBUG }, 19:45:52 DEBUG "status": "success", 19:45:52 DEBUG }, 19:45:52 DEBUG "value": [ 19:45:52 DEBUG "data": { 19:45:52 DEBUG "value": [ 19:45:52 DEBUG 1713638752, 19:45:52 DEBUG "resultType": "vector", 19:45:52 DEBUG 1713638752, 19:45:52 DEBUG "0" 19:45:52 DEBUG "result": [ 19:45:52 DEBUG "2672.933333333334" 19:45:52 DEBUG ] 19:45:52 DEBUG { 19:45:52 DEBUG ] 19:45:52 DEBUG } 19:45:52 DEBUG "metric": { 19:45:52 DEBUG } 19:45:52 DEBUG ] 19:45:52 DEBUG "pod": "ds-idrepo-0" 19:45:52 DEBUG ] 19:45:52 DEBUG } 19:45:52 DEBUG }, 19:45:52 DEBUG } 19:45:52 DEBUG } 19:45:52 DEBUG "value": [ 19:45:52 DEBUG } 19:45:52 DEBUG 19:45:52 DEBUG 19:45:52 DEBUG 19:45:52 DEBUG [...] 19:45:52 DEBUG 19:45:52 DEBUG 19:45:52 DEBUG 19:45:52 DEBUG "pod": "ds-idrepo-2" 19:45:52 DEBUG }, 19:45:52 DEBUG "value": [ 19:45:52 DEBUG 1713638752, 19:45:52 DEBUG "0" 19:45:52 DEBUG ] 19:45:52 DEBUG } 19:45:52 DEBUG ] 19:45:52 DEBUG } 19:45:52 DEBUG } 19:45:53 INFO 19:45:53 INFO [loop_until]: kubectl --namespace=xlou top pods 19:45:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:45:53 INFO [loop_until]: OK (rc = 0) 19:45:53 DEBUG --- stdout --- 19:45:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16763m 8655Mi am-767f6b67b5-qcqz2 17255m 9065Mi am-767f6b67b5-vbltz 18688m 8667Mi ds-cts-0 18964m 10247Mi ds-cts-1 17518m 10201Mi ds-cts-2 19765m 10255Mi ds-idrepo-0 7215m 3888Mi ds-idrepo-1 7255m 3882Mi ds-idrepo-2 7434m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8843m 744Mi 19:45:53 DEBUG --- stderr --- 19:45:53 DEBUG 19:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638753" 19:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638753" 19:45:53 INFO [http_cmd]: http status code OK 19:45:53 DEBUG --- status code --- 19:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:53 DEBUG --- http response --- 19:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:53 DEBUG ----- output ----- 19:45:53 DEBUG { 19:45:53 DEBUG "status": "success", 19:45:53 DEBUG "data": { 19:45:53 DEBUG "resultType": "vector", 19:45:53 DEBUG "result": [ 19:45:53 DEBUG { 19:45:53 DEBUG "metric": { 19:45:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:53 DEBUG }, 19:45:53 DEBUG "value": [ 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG [...] 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:53 DEBUG }, 19:45:53 DEBUG "value": [ 19:45:53 DEBUG 1713638753, 19:45:53 DEBUG "2666.4" 19:45:53 DEBUG ] 19:45:53 DEBUG } 19:45:53 DEBUG ] 19:45:53 DEBUG } 19:45:53 DEBUG } 19:45:53 INFO [http_cmd]: http status code OK 19:45:53 DEBUG --- status code --- 19:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:53 DEBUG --- http response --- 19:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:45:53 DEBUG ----- output ----- 19:45:53 DEBUG { 19:45:53 DEBUG "status": "success", 19:45:53 DEBUG "data": { 19:45:53 DEBUG "resultType": "vector", 19:45:53 DEBUG "result": [ 19:45:53 DEBUG { 19:45:53 DEBUG "metric": { 19:45:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:45:53 DEBUG }, 19:45:53 DEBUG "value": [ 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG [...] 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:53 DEBUG }, 19:45:53 DEBUG "value": [ 19:45:53 DEBUG 1713638753, 19:45:53 DEBUG "0" 19:45:53 DEBUG ] 19:45:53 DEBUG } 19:45:53 DEBUG ] 19:45:53 DEBUG } 19:45:53 DEBUG } 19:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638753" 19:45:53 INFO [http_cmd]: http status code OK 19:45:53 DEBUG --- status code --- 19:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:53 DEBUG --- http response --- 19:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:53 DEBUG ----- output ----- 19:45:53 DEBUG { 19:45:53 DEBUG "status": "success", 19:45:53 DEBUG "data": { 19:45:53 DEBUG "resultType": "vector", 19:45:53 DEBUG "result": [ 19:45:53 DEBUG { 19:45:53 DEBUG "metric": { 19:45:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:45:53 DEBUG }, 19:45:53 DEBUG "value": [ 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG [...] 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:45:53 DEBUG }, 19:45:53 DEBUG "value": [ 19:45:53 DEBUG 1713638753, 19:45:53 DEBUG "8.883874041885921" 19:45:53 DEBUG ] 19:45:53 DEBUG } 19:45:53 DEBUG ] 19:45:53 DEBUG } 19:45:53 DEBUG } 19:45:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638753" 19:45:53 INFO [http_cmd]: http status code OK 19:45:53 DEBUG --- status code --- 19:45:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:53 DEBUG --- http response --- 19:45:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:53 DEBUG ----- output ----- 19:45:53 DEBUG { 19:45:53 DEBUG "status": "success", 19:45:53 DEBUG "data": { 19:45:53 DEBUG "resultType": "vector", 19:45:53 DEBUG "result": [ 19:45:53 DEBUG { 19:45:53 DEBUG "metric": { 19:45:53 DEBUG "instance": "10.142.0.107:9100" 19:45:53 DEBUG }, 19:45:53 DEBUG "value": [ 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG [...] 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG 19:45:53 DEBUG "instance": "10.142.0.43:9100" 19:45:53 DEBUG }, 19:45:53 DEBUG "value": [ 19:45:53 DEBUG 1713638753, 19:45:53 DEBUG "3590460695552" 19:45:53 DEBUG ] 19:45:53 DEBUG } 19:45:53 DEBUG ] 19:45:53 DEBUG } 19:45:53 DEBUG } 19:45:54 INFO 19:45:54 INFO [loop_until]: kubectl --namespace=xlou top node 19:45:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:45:54 INFO [loop_until]: OK (rc = 0) 19:45:54 DEBUG --- stdout --- 19:45:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6208Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7545m 25% 7636Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16944m 56% 11365Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 96m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18756m 62% 13483Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5186Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17307m 57% 11951Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14545m 48% 2956Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19498m 65% 10892Mi 9% gke-xlou-cdm-ds-0f601751-j488 8156m 27% 7190Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17006m 56% 15284Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8085m 27% 7869Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19494m 65% 12917Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9081m 57% 6368Mi 10% 19:45:54 DEBUG --- stderr --- 19:45:54 DEBUG 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG { 19:45:54 DEBUG "status": "success", 19:45:54 DEBUG "data": { 19:45:54 DEBUG "resultType": "vector", 19:45:54 DEBUG "result": [ 19:45:54 DEBUG { 19:45:54 DEBUG "metric": { 19:45:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:54 DEBUG }, 19:45:54 DEBUG "value": [ 19:45:54 DEBUG 19:45:54 DEBUG 19:45:54 DEBUG 19:45:54 DEBUG [...] 19:45:54 DEBUG 19:45:54 DEBUG 19:45:54 DEBUG 19:45:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:54 DEBUG }, 19:45:54 DEBUG "value": [ 19:45:54 DEBUG 1713638754, 19:45:54 DEBUG "2668.4333333333334" 19:45:54 DEBUG ] 19:45:54 DEBUG } 19:45:54 DEBUG ] 19:45:54 DEBUG } 19:45:54 DEBUG } 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG { 19:45:54 DEBUG "status": "success", 19:45:54 DEBUG "data": { 19:45:54 DEBUG "resultType": "vector", 19:45:54 DEBUG "result": [ 19:45:54 DEBUG { 19:45:54 DEBUG "metric": { 19:45:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:54 DEBUG }, 19:45:54 DEBUG "value": [ 19:45:54 DEBUG 19:45:54 DEBUG 19:45:54 DEBUG 19:45:54 DEBUG [...] 19:45:54 DEBUG 19:45:54 DEBUG 19:45:54 DEBUG 19:45:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:54 DEBUG }, 19:45:54 DEBUG "value": [ 19:45:54 DEBUG 1713638754, 19:45:54 DEBUG "2665.8" 19:45:54 DEBUG ] 19:45:54 DEBUG } 19:45:54 DEBUG ] 19:45:54 DEBUG } 19:45:54 DEBUG } 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG { 19:45:54 DEBUG "status": "success", 19:45:54 DEBUG "data": { 19:45:54 DEBUG "resultType": "vector", 19:45:54 DEBUG "result": [ 19:45:54 DEBUG { 19:45:54 DEBUG "metric": { 19:45:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:45:54 DEBUG }, 19:45:54 DEBUG "value": [ 19:45:54 DEBUG 19:45:54 DEBUG 19:45:54 DEBUG 19:45:54 DEBUG [...] 19:45:54 DEBUG 19:45:54 DEBUG 19:45:54 DEBUG 19:45:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:54 DEBUG }, 19:45:54 DEBUG "value": [ 19:45:54 DEBUG 1713638754, 19:45:54 DEBUG "9.991584062533038" 19:45:54 DEBUG ] 19:45:54 DEBUG } 19:45:54 DEBUG ] 19:45:54 DEBUG } 19:45:54 DEBUG } 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638754" 19:45:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638754" 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 DEBUG --- status code --- 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 DEBUG --- status code --- 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG --- status code --- 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:54 INFO [http_cmd]: http status code OK 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG --- status code --- 19:45:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:45:54 DEBUG --- http response --- 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:54 DEBUG { 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG { 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG { 19:45:54 DEBUG { 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:45:54 DEBUG "status": "success", 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG { 19:45:54 DEBUG "status": "success", 19:45:54 DEBUG { 19:45:54 DEBUG { 19:45:54 DEBUG { 19:45:54 DEBUG { 19:45:54 DEBUG "status": "success", 19:45:54 DEBUG "status": "success", 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG { 19:45:54 DEBUG { 19:45:54 DEBUG ----- output ----- 19:45:54 DEBUG "data": { 19:45:54 DEBUG { 19:45:54 DEBUG { 19:45:54 DEBUG "status": "success", 19:45:54 DEBUG "data": { 19:45:55 DEBUG "resultType": "vector", 19:45:54 DEBUG "status": "success", 19:45:54 DEBUG "status": "success", 19:45:54 DEBUG "status": "success", 19:45:54 DEBUG "data": { 19:45:54 DEBUG "data": { 19:45:55 DEBUG { 19:45:55 DEBUG "status": "success", 19:45:55 DEBUG "status": "success", 19:45:55 DEBUG { 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG "status": "success", 19:45:55 DEBUG "status": "success", 19:45:55 DEBUG "data": { 19:45:55 DEBUG "result": [ 19:45:54 DEBUG "status": "success", 19:45:55 DEBUG "data": { 19:45:55 DEBUG "data": { 19:45:55 DEBUG "data": { 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG "status": "success", 19:45:55 DEBUG "data": { 19:45:55 DEBUG "data": { 19:45:55 DEBUG "status": "success", 19:45:55 DEBUG "data": { 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG "data": { 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG "result": [ 19:45:55 DEBUG "data": { 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG "result": [ 19:45:55 DEBUG "result": [ 19:45:55 DEBUG "data": { 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG "data": { 19:45:55 DEBUG "result": [ 19:45:55 DEBUG "result": [ 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG { 19:45:55 DEBUG { 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG "result": [ 19:45:55 DEBUG "result": [ 19:45:55 DEBUG "result": [ 19:45:55 DEBUG { 19:45:55 DEBUG { 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG "result": [ 19:45:55 DEBUG "result": [ 19:45:55 DEBUG "resultType": "vector", 19:45:55 DEBUG { 19:45:55 DEBUG { 19:45:55 DEBUG "result": [ 19:45:55 DEBUG "metric": { 19:45:55 DEBUG "metric": { 19:45:55 DEBUG "result": [ 19:45:55 DEBUG { 19:45:55 DEBUG { 19:45:55 DEBUG { 19:45:55 DEBUG "metric": { 19:45:55 DEBUG "metric": { 19:45:55 DEBUG "result": [ 19:45:55 DEBUG { 19:45:55 DEBUG { 19:45:55 DEBUG "result": [ 19:45:55 DEBUG "metric": { 19:45:55 DEBUG "metric": { 19:45:55 DEBUG { 19:45:55 DEBUG "instance": "10.142.0.107:9100" 19:45:55 DEBUG "pod": "ds-idrepo-0" 19:45:55 DEBUG { 19:45:55 DEBUG "metric": { 19:45:55 DEBUG "metric": { 19:45:55 DEBUG "metric": { 19:45:55 DEBUG "pod": "ds-cts-0" 19:45:55 DEBUG }, 19:45:55 DEBUG { 19:45:55 DEBUG "metric": { 19:45:55 DEBUG "metric": { 19:45:55 DEBUG { 19:45:55 DEBUG "instance": "10.142.0.107:9100" 19:45:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:45:55 DEBUG "metric": { 19:45:55 DEBUG }, 19:45:55 DEBUG }, 19:45:55 DEBUG "metric": { 19:45:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:55 DEBUG "instance": "10.142.0.107:9100" 19:45:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:45:55 DEBUG "value": [ 19:45:55 DEBUG "metric": { 19:45:55 DEBUG "pod": "ds-cts-0" 19:45:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:45:55 DEBUG "metric": { 19:45:55 DEBUG }, 19:45:55 DEBUG }, 19:45:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:55 DEBUG "value": [ 19:45:55 DEBUG "value": [ 19:45:55 DEBUG "pod": "ds-cts-0" 19:45:55 DEBUG }, 19:45:55 DEBUG }, 19:45:55 DEBUG }, 19:45:55 DEBUG }, 19:45:55 DEBUG 19:45:55 DEBUG }, 19:45:55 DEBUG "pod": "ds-idrepo-0" 19:45:55 DEBUG }, 19:45:55 DEBUG "instance": "10.142.0.107:9100" 19:45:55 DEBUG "value": [ 19:45:55 DEBUG "value": [ 19:45:55 DEBUG }, 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG }, 19:45:55 DEBUG "value": [ 19:45:55 DEBUG "value": [ 19:45:55 DEBUG "value": [ 19:45:55 DEBUG "value": [ 19:45:55 DEBUG 19:45:55 DEBUG "value": [ 19:45:55 DEBUG }, 19:45:55 DEBUG "value": [ 19:45:55 DEBUG }, 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG "value": [ 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG "value": [ 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG "value": [ 19:45:55 DEBUG 19:45:55 DEBUG "value": [ 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG [...] 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG [...] 19:45:55 DEBUG [...] 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG [...] 19:45:55 DEBUG [...] 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG [...] 19:45:55 DEBUG [...] 19:45:55 DEBUG [...] 19:45:55 DEBUG [...] 19:45:55 DEBUG 19:45:55 DEBUG [...] 19:45:55 DEBUG 19:45:55 DEBUG [...] 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG [...] 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG [...] 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG [...] 19:45:55 DEBUG 19:45:55 DEBUG [...] 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG "instance": "10.142.0.43:9100" 19:45:55 DEBUG "pod": "ds-cts-2" 19:45:55 DEBUG }, 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG "pod": "ds-cts-2" 19:45:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:55 DEBUG }, 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG 19:45:55 DEBUG "instance": "10.142.0.43:9100" 19:45:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:45:55 DEBUG 19:45:55 DEBUG }, 19:45:55 DEBUG "value": [ 19:45:55 DEBUG 19:45:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:55 DEBUG "instance": "10.142.0.43:9100" 19:45:55 DEBUG }, 19:45:55 DEBUG 19:45:55 DEBUG "value": [ 19:45:55 DEBUG 19:45:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:45:55 DEBUG 19:45:55 DEBUG }, 19:45:55 DEBUG }, 19:45:55 DEBUG "value": [ 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG }, 19:45:55 DEBUG 19:45:55 DEBUG "pod": "ds-cts-2" 19:45:55 DEBUG }, 19:45:55 DEBUG "value": [ 19:45:55 DEBUG "pod": "ds-cts-2" 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG "pod": "ds-idrepo-2" 19:45:55 DEBUG }, 19:45:55 DEBUG "instance": "10.142.0.43:9100" 19:45:55 DEBUG "value": [ 19:45:55 DEBUG "0" 19:45:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:45:55 DEBUG "value": [ 19:45:55 DEBUG "0" 19:45:55 DEBUG "value": [ 19:45:55 DEBUG }, 19:45:55 DEBUG }, 19:45:55 DEBUG "value": [ 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG }, 19:45:55 DEBUG "927.4857685337017" 19:45:55 DEBUG }, 19:45:55 DEBUG "value": [ 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG ] 19:45:55 DEBUG }, 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG ] 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG "value": [ 19:45:55 DEBUG "value": [ 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG "30" 19:45:55 DEBUG "value": [ 19:45:55 DEBUG ] 19:45:55 DEBUG "value": [ 19:45:55 DEBUG }, 19:45:55 DEBUG "876904448" 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG "value": [ 19:45:55 DEBUG "80876475.1" 19:45:55 DEBUG } 19:45:55 DEBUG "0" 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG "0.1545" 19:45:55 DEBUG ] 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG } 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG "value": [ 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG "80873395.69999999" 19:45:55 DEBUG ] 19:45:55 DEBUG 1713638754, 19:45:55 DEBUG ] 19:45:55 DEBUG ] 19:45:55 DEBUG ] 19:45:55 DEBUG "0.17606666666664048" 19:45:55 DEBUG "0.00006666666666666672" 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG "0" 19:45:55 DEBUG ] 19:45:55 DEBUG "0" 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG "0.06899999999999028" 19:45:55 DEBUG } 19:45:55 DEBUG "480258.5" 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG ] 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG ] 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:45:55 DEBUG } 19:46:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638811" 19:46:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638811" 19:46:51 ERROR [http_cmd]: ERROR 19:46:51 ERROR --- status code --- 19:46:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:46:51 ERROR --- http response --- 19:46:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:46:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:46:51 INFO [http_cmd]: http status code OK 19:46:51 DEBUG --- status code --- 19:46:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:51 DEBUG --- http response --- 19:46:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:46:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638812" 19:46:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638812" 19:46:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638812" 19:46:53 INFO [http_cmd]: http status code OK 19:46:53 INFO [http_cmd]: http status code OK 19:46:53 INFO [http_cmd]: http status code OK 19:46:53 DEBUG --- status code --- 19:46:53 DEBUG --- status code --- 19:46:53 DEBUG --- status code --- 19:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:53 DEBUG --- http response --- 19:46:53 DEBUG --- http response --- 19:46:53 DEBUG --- http response --- 19:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:53 DEBUG ----- output ----- 19:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:53 DEBUG ----- output ----- 19:46:53 DEBUG { 19:46:53 DEBUG ----- output ----- 19:46:53 DEBUG { 19:46:53 DEBUG "status": "success", 19:46:53 DEBUG { 19:46:53 DEBUG "status": "success", 19:46:53 DEBUG "data": { 19:46:53 DEBUG "status": "success", 19:46:53 DEBUG "data": { 19:46:53 DEBUG "resultType": "vector", 19:46:53 DEBUG "data": { 19:46:53 DEBUG "resultType": "vector", 19:46:53 DEBUG "result": [ 19:46:53 DEBUG "resultType": "vector", 19:46:53 DEBUG "result": [ 19:46:53 DEBUG { 19:46:53 DEBUG "result": [ 19:46:53 DEBUG { 19:46:53 DEBUG "metric": { 19:46:53 DEBUG "pod": "ds-idrepo-0" 19:46:53 DEBUG "metric": { 19:46:53 DEBUG }, 19:46:53 DEBUG { 19:46:53 DEBUG "pod": "ds-idrepo-0" 19:46:53 DEBUG "value": [ 19:46:53 DEBUG "metric": { 19:46:53 DEBUG }, 19:46:53 DEBUG 19:46:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:53 DEBUG "value": [ 19:46:53 DEBUG 19:46:53 DEBUG }, 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG "value": [ 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG [...] 19:46:53 DEBUG [...] 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG [...] 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG "pod": "ds-idrepo-2" 19:46:53 DEBUG "pod": "ds-idrepo-2" 19:46:53 DEBUG 19:46:53 DEBUG }, 19:46:53 DEBUG }, 19:46:53 DEBUG 19:46:53 DEBUG "value": [ 19:46:53 DEBUG "value": [ 19:46:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:53 DEBUG 1713638812, 19:46:53 DEBUG 1713638812, 19:46:53 DEBUG }, 19:46:53 DEBUG "0" 19:46:53 DEBUG "0" 19:46:53 DEBUG "value": [ 19:46:53 DEBUG ] 19:46:53 DEBUG ] 19:46:53 DEBUG 1713638812, 19:46:53 DEBUG } 19:46:53 DEBUG } 19:46:53 DEBUG "2669.5000000000005" 19:46:53 DEBUG ] 19:46:53 DEBUG ] 19:46:53 DEBUG ] 19:46:53 DEBUG } 19:46:53 DEBUG } 19:46:53 DEBUG } 19:46:53 DEBUG } 19:46:53 DEBUG } 19:46:53 DEBUG ] 19:46:53 DEBUG } 19:46:53 DEBUG } 19:46:53 INFO 19:46:53 INFO [loop_until]: kubectl --namespace=xlou top pods 19:46:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:46:53 INFO [loop_until]: OK (rc = 0) 19:46:53 DEBUG --- stdout --- 19:46:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16528m 8651Mi am-767f6b67b5-qcqz2 16998m 9067Mi am-767f6b67b5-vbltz 18691m 8664Mi ds-cts-0 19374m 10239Mi ds-cts-1 17217m 10208Mi ds-cts-2 19136m 10261Mi ds-idrepo-0 7290m 3888Mi ds-idrepo-1 7227m 3883Mi ds-idrepo-2 7298m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 12m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8894m 745Mi 19:46:53 DEBUG --- stderr --- 19:46:53 DEBUG 19:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638813" 19:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638813" 19:46:53 INFO [http_cmd]: http status code OK 19:46:53 DEBUG --- status code --- 19:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:53 DEBUG --- http response --- 19:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:53 DEBUG ----- output ----- 19:46:53 DEBUG { 19:46:53 DEBUG "status": "success", 19:46:53 DEBUG "data": { 19:46:53 DEBUG "resultType": "vector", 19:46:53 DEBUG "result": [ 19:46:53 DEBUG { 19:46:53 DEBUG "metric": { 19:46:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:53 DEBUG }, 19:46:53 DEBUG "value": [ 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG [...] 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:53 DEBUG }, 19:46:53 DEBUG "value": [ 19:46:53 DEBUG 1713638813, 19:46:53 DEBUG "2659.4333333333334" 19:46:53 DEBUG ] 19:46:53 DEBUG } 19:46:53 DEBUG ] 19:46:53 DEBUG } 19:46:53 DEBUG } 19:46:53 INFO [http_cmd]: http status code OK 19:46:53 DEBUG --- status code --- 19:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:53 DEBUG --- http response --- 19:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:53 DEBUG ----- output ----- 19:46:53 DEBUG { 19:46:53 DEBUG "status": "success", 19:46:53 DEBUG "data": { 19:46:53 DEBUG "resultType": "vector", 19:46:53 DEBUG "result": [ 19:46:53 DEBUG { 19:46:53 DEBUG "metric": { 19:46:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:46:53 DEBUG }, 19:46:53 DEBUG "value": [ 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG [...] 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG "pod": "ds-cts-0" 19:46:53 DEBUG }, 19:46:53 DEBUG "value": [ 19:46:53 DEBUG 1713638813, 19:46:53 DEBUG "70114046.95983604" 19:46:53 DEBUG ] 19:46:53 DEBUG } 19:46:53 DEBUG ] 19:46:53 DEBUG } 19:46:53 DEBUG } 19:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638813" 19:46:53 INFO [http_cmd]: http status code OK 19:46:53 DEBUG --- status code --- 19:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:53 DEBUG --- http response --- 19:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:53 DEBUG ----- output ----- 19:46:53 DEBUG { 19:46:53 DEBUG "status": "success", 19:46:53 DEBUG "data": { 19:46:53 DEBUG "resultType": "vector", 19:46:53 DEBUG "result": [ 19:46:53 DEBUG { 19:46:53 DEBUG "metric": { 19:46:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:46:53 DEBUG }, 19:46:53 DEBUG "value": [ 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG [...] 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:46:53 DEBUG }, 19:46:53 DEBUG "value": [ 19:46:53 DEBUG 1713638813, 19:46:53 DEBUG "8.859367693844307" 19:46:53 DEBUG ] 19:46:53 DEBUG } 19:46:53 DEBUG ] 19:46:53 DEBUG } 19:46:53 DEBUG } 19:46:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638813" 19:46:53 INFO [http_cmd]: http status code OK 19:46:53 DEBUG --- status code --- 19:46:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:53 DEBUG --- http response --- 19:46:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:53 DEBUG ----- output ----- 19:46:53 DEBUG { 19:46:53 DEBUG "status": "success", 19:46:53 DEBUG "data": { 19:46:53 DEBUG "resultType": "vector", 19:46:53 DEBUG "result": [ 19:46:53 DEBUG { 19:46:53 DEBUG "metric": { 19:46:53 DEBUG "instance": "10.142.0.107:9100" 19:46:53 DEBUG }, 19:46:53 DEBUG "value": [ 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG [...] 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG 19:46:53 DEBUG "instance": "10.142.0.43:9100" 19:46:53 DEBUG }, 19:46:53 DEBUG "value": [ 19:46:53 DEBUG 1713638813, 19:46:53 DEBUG "3594480907264" 19:46:53 DEBUG ] 19:46:53 DEBUG } 19:46:53 DEBUG ] 19:46:53 DEBUG } 19:46:53 DEBUG } 19:46:54 INFO 19:46:54 INFO [loop_until]: kubectl --namespace=xlou top node 19:46:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:46:54 INFO [loop_until]: OK (rc = 0) 19:46:54 DEBUG --- stdout --- 19:46:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6199Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7530m 25% 7636Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16816m 56% 11379Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3346Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19152m 64% 13489Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5199Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17154m 57% 11954Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14486m 48% 2965Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19468m 65% 10888Mi 9% gke-xlou-cdm-ds-0f601751-j488 8132m 27% 7180Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17106m 57% 15268Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8046m 26% 7852Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19006m 63% 12939Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9051m 56% 6385Mi 10% 19:46:54 DEBUG --- stderr --- 19:46:54 DEBUG 19:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638814" 19:46:54 INFO [http_cmd]: http status code OK 19:46:54 DEBUG --- status code --- 19:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:54 DEBUG --- http response --- 19:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:54 DEBUG ----- output ----- 19:46:54 DEBUG { 19:46:54 DEBUG "status": "success", 19:46:54 DEBUG "data": { 19:46:54 DEBUG "resultType": "vector", 19:46:54 DEBUG "result": [ 19:46:54 DEBUG { 19:46:54 DEBUG "metric": { 19:46:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:54 DEBUG }, 19:46:54 DEBUG "value": [ 19:46:54 DEBUG 19:46:54 DEBUG 19:46:54 DEBUG 19:46:54 DEBUG [...] 19:46:54 DEBUG 19:46:54 DEBUG 19:46:54 DEBUG 19:46:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:54 DEBUG }, 19:46:54 DEBUG "value": [ 19:46:54 DEBUG 1713638814, 19:46:54 DEBUG "2659.8333333333335" 19:46:54 DEBUG ] 19:46:54 DEBUG } 19:46:54 DEBUG ] 19:46:54 DEBUG } 19:46:54 DEBUG } 19:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638814" 19:46:54 INFO [http_cmd]: http status code OK 19:46:54 DEBUG --- status code --- 19:46:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:54 DEBUG --- http response --- 19:46:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:54 DEBUG ----- output ----- 19:46:54 DEBUG { 19:46:54 DEBUG "status": "success", 19:46:54 DEBUG "data": { 19:46:54 DEBUG "resultType": "vector", 19:46:54 DEBUG "result": [ 19:46:54 DEBUG { 19:46:54 DEBUG "metric": { 19:46:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:54 DEBUG }, 19:46:54 DEBUG "value": [ 19:46:54 DEBUG 19:46:54 DEBUG 19:46:54 DEBUG 19:46:54 DEBUG [...] 19:46:54 DEBUG 19:46:54 DEBUG 19:46:54 DEBUG 19:46:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:54 DEBUG }, 19:46:54 DEBUG "value": [ 19:46:54 DEBUG 1713638814, 19:46:54 DEBUG "2661.1666666666665" 19:46:54 DEBUG ] 19:46:54 DEBUG } 19:46:54 DEBUG ] 19:46:54 DEBUG } 19:46:54 DEBUG } 19:46:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638814" 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG { 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG "data": { 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG "result": [ 19:46:55 DEBUG { 19:46:55 DEBUG "metric": { 19:46:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG "pod": "ds-cts-0" 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 1713638814, 19:46:55 DEBUG "19.046013088348545" 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638815" 19:46:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638815" 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 DEBUG --- status code --- 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 DEBUG --- status code --- 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 DEBUG --- status code --- 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG --- http response --- 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG --- http response --- 19:46:55 INFO [http_cmd]: http status code OK 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG --- status code --- 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:46:55 DEBUG { 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:46:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:46:55 DEBUG { 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG --- http response --- 19:46:55 DEBUG { 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG { 19:46:55 DEBUG { 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:55 DEBUG { 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG { 19:46:55 DEBUG "data": { 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG { 19:46:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG { 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG { 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG "data": { 19:46:55 DEBUG { 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG { 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG "data": { 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG "result": [ 19:46:55 DEBUG { 19:46:55 DEBUG "metric": { 19:46:55 DEBUG { 19:46:55 DEBUG "data": { 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG { 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG "data": { 19:46:55 DEBUG "result": [ 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG "data": { 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG "data": { 19:46:55 DEBUG "pod": "ds-cts-0" 19:46:55 DEBUG "data": { 19:46:55 DEBUG ----- output ----- 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG "data": { 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG "data": { 19:46:55 DEBUG "data": { 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG { 19:46:55 DEBUG "data": { 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG "data": { 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG { 19:46:55 DEBUG "data": { 19:46:55 DEBUG "result": [ 19:46:55 DEBUG { 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG "result": [ 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG "result": [ 19:46:55 DEBUG { 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG "result": [ 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG "result": [ 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG "metric": { 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG "result": [ 19:46:55 DEBUG { 19:46:55 DEBUG "result": [ 19:46:55 DEBUG "metric": { 19:46:55 DEBUG "pod": "ds-cts-0" 19:46:55 DEBUG "result": [ 19:46:55 DEBUG { 19:46:55 DEBUG "result": [ 19:46:55 DEBUG { 19:46:55 DEBUG "result": [ 19:46:55 DEBUG [...] 19:46:55 DEBUG "status": "success", 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:55 DEBUG "result": [ 19:46:55 DEBUG { 19:46:55 DEBUG "metric": { 19:46:55 DEBUG { 19:46:55 DEBUG "metric": { 19:46:55 DEBUG }, 19:46:55 DEBUG { 19:46:55 DEBUG "metric": { 19:46:55 DEBUG { 19:46:55 DEBUG "metric": { 19:46:55 DEBUG { 19:46:55 DEBUG 19:46:55 DEBUG "data": { 19:46:55 DEBUG "result": [ 19:46:55 DEBUG }, 19:46:55 DEBUG { 19:46:55 DEBUG "metric": { 19:46:55 DEBUG "pod": "ds-idrepo-0" 19:46:55 DEBUG "metric": { 19:46:55 DEBUG "instance": "10.142.0.107:9100" 19:46:55 DEBUG "value": [ 19:46:55 DEBUG "metric": { 19:46:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:46:55 DEBUG "metric": { 19:46:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:46:55 DEBUG "metric": { 19:46:55 DEBUG 19:46:55 DEBUG "resultType": "vector", 19:46:55 DEBUG { 19:46:55 DEBUG "value": [ 19:46:55 DEBUG "metric": { 19:46:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:55 DEBUG }, 19:46:55 DEBUG "instance": "10.142.0.107:9100" 19:46:55 DEBUG }, 19:46:55 DEBUG 19:46:55 DEBUG "pod": "ds-cts-0" 19:46:55 DEBUG }, 19:46:55 DEBUG "instance": "10.142.0.107:9100" 19:46:55 DEBUG }, 19:46:55 DEBUG "pod": "ds-idrepo-0" 19:46:55 DEBUG 19:46:55 DEBUG "result": [ 19:46:55 DEBUG "metric": { 19:46:55 DEBUG 19:46:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG }, 19:46:55 DEBUG "pod": "ds-cts-2" 19:46:55 DEBUG { 19:46:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:46:55 DEBUG 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 19:46:55 DEBUG "value": [ 19:46:55 DEBUG }, 19:46:55 DEBUG "metric": { 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG "value": [ 19:46:55 DEBUG "instance": "10.142.0.107:9100" 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG }, 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG "0.000033333333333333335" 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG [...] 19:46:55 DEBUG ] 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG [...] 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG "pod": "ds-cts-2" 19:46:55 DEBUG 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG } 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG }, 19:46:55 DEBUG "pod": "ds-cts-2" 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:46:55 DEBUG ] 19:46:55 DEBUG [...] 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG "pod": "ds-cts-0" 19:46:55 DEBUG 19:46:55 DEBUG "instance": "10.142.0.43:9100" 19:46:55 DEBUG "instance": "10.142.0.43:9100" 19:46:55 DEBUG "value": [ 19:46:55 DEBUG }, 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG 19:46:55 DEBUG }, 19:46:55 DEBUG } 19:46:55 DEBUG 19:46:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:55 DEBUG 19:46:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:55 DEBUG }, 19:46:55 DEBUG }, 19:46:55 DEBUG 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 19:46:55 DEBUG "instance": "10.142.0.43:9100" 19:46:55 DEBUG 19:46:55 DEBUG "value": [ 19:46:55 DEBUG } 19:46:55 DEBUG 19:46:55 DEBUG }, 19:46:55 DEBUG 19:46:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG "pod": "ds-idrepo-2" 19:46:55 DEBUG }, 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG "pod": "ds-cts-0" 19:46:55 DEBUG }, 19:46:55 DEBUG "pod": "ds-cts-2" 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG 19:46:55 DEBUG "value": [ 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG "value": [ 19:46:55 DEBUG "29.9" 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG "0" 19:46:55 DEBUG }, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG }, 19:46:55 DEBUG "878768128" 19:46:55 DEBUG "instance": "10.142.0.43:9100" 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG "80882664.3" 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG ] 19:46:55 DEBUG "80877980.13333333" 19:46:55 DEBUG ] 19:46:55 DEBUG "value": [ 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG "value": [ 19:46:55 DEBUG ] 19:46:55 DEBUG }, 19:46:55 DEBUG "0.155" 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG "0" 19:46:55 DEBUG ] 19:46:55 DEBUG "0" 19:46:55 DEBUG "54717705.65603538" 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG "0.09466666666665256" 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG } 19:46:55 DEBUG "value": [ 19:46:55 DEBUG ] 19:46:55 DEBUG "479701" 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG ] 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG "386586933.8031714" 19:46:55 DEBUG ] 19:46:55 DEBUG "0" 19:46:55 DEBUG ] 19:46:55 DEBUG 1713638815, 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG "0.2563666666666563" 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG ] 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:46:55 DEBUG } 19:47:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638871" 19:47:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638871" 19:47:51 ERROR [http_cmd]: ERROR 19:47:51 ERROR --- status code --- 19:47:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:47:51 ERROR --- http response --- 19:47:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:47:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:47:51 INFO [http_cmd]: http status code OK 19:47:51 DEBUG --- status code --- 19:47:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:51 DEBUG --- http response --- 19:47:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638873" 19:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638873" 19:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638873" 19:47:53 INFO [http_cmd]: http status code OK 19:47:53 INFO [http_cmd]: http status code OK 19:47:53 DEBUG --- status code --- 19:47:53 DEBUG --- status code --- 19:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:53 DEBUG --- http response --- 19:47:53 DEBUG --- http response --- 19:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:53 DEBUG ----- output ----- 19:47:53 DEBUG { 19:47:53 DEBUG "status": "success", 19:47:53 DEBUG "data": { 19:47:53 DEBUG "resultType": "vector", 19:47:53 DEBUG "result": [ 19:47:53 DEBUG { 19:47:53 DEBUG "metric": { 19:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:53 DEBUG "pod": "ds-idrepo-0" 19:47:53 DEBUG ----- output ----- 19:47:53 DEBUG }, 19:47:53 DEBUG { 19:47:53 DEBUG "value": [ 19:47:53 DEBUG "status": "success", 19:47:53 INFO [http_cmd]: http status code OK 19:47:53 DEBUG 19:47:53 DEBUG "data": { 19:47:53 DEBUG --- status code --- 19:47:53 DEBUG 19:47:53 DEBUG "resultType": "vector", 19:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:53 DEBUG 19:47:53 DEBUG "result": [ 19:47:53 DEBUG --- http response --- 19:47:53 DEBUG [...] 19:47:53 DEBUG { 19:47:53 DEBUG 19:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:53 DEBUG "metric": { 19:47:53 DEBUG 19:47:53 DEBUG ----- output ----- 19:47:53 DEBUG "pod": "ds-idrepo-0" 19:47:53 DEBUG 19:47:53 DEBUG { 19:47:53 DEBUG }, 19:47:53 DEBUG "pod": "ds-idrepo-2" 19:47:53 DEBUG "status": "success", 19:47:53 DEBUG "value": [ 19:47:53 DEBUG }, 19:47:53 DEBUG "data": { 19:47:53 DEBUG 19:47:53 DEBUG "value": [ 19:47:53 DEBUG "resultType": "vector", 19:47:53 DEBUG 19:47:53 DEBUG 1713638873, 19:47:53 DEBUG "result": [ 19:47:53 DEBUG 19:47:53 DEBUG "0" 19:47:53 DEBUG { 19:47:53 DEBUG [...] 19:47:53 DEBUG ] 19:47:53 DEBUG "metric": { 19:47:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:53 DEBUG } 19:47:53 DEBUG 19:47:53 DEBUG }, 19:47:53 DEBUG "value": [ 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG ] 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG } 19:47:53 DEBUG "pod": "ds-idrepo-2" 19:47:53 DEBUG 19:47:53 DEBUG } 19:47:53 DEBUG }, 19:47:53 DEBUG [...] 19:47:53 DEBUG "value": [ 19:47:53 DEBUG 19:47:53 DEBUG 1713638873, 19:47:53 DEBUG 19:47:53 DEBUG "0" 19:47:53 DEBUG 19:47:53 DEBUG ] 19:47:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:53 DEBUG } 19:47:53 DEBUG }, 19:47:53 DEBUG ] 19:47:53 DEBUG "value": [ 19:47:53 DEBUG } 19:47:53 DEBUG 1713638873, 19:47:53 DEBUG } 19:47:53 DEBUG "2669.6" 19:47:53 DEBUG ] 19:47:53 DEBUG } 19:47:53 DEBUG ] 19:47:53 DEBUG } 19:47:53 DEBUG } 19:47:53 INFO 19:47:53 INFO [loop_until]: kubectl --namespace=xlou top pods 19:47:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:47:53 INFO [loop_until]: OK (rc = 0) 19:47:53 DEBUG --- stdout --- 19:47:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16815m 8649Mi am-767f6b67b5-qcqz2 17103m 9064Mi am-767f6b67b5-vbltz 18606m 8665Mi ds-cts-0 18755m 10254Mi ds-cts-1 17061m 10209Mi ds-cts-2 19141m 10262Mi ds-idrepo-0 7290m 3888Mi ds-idrepo-1 7409m 3882Mi ds-idrepo-2 7211m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 12m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8818m 748Mi 19:47:53 DEBUG --- stderr --- 19:47:53 DEBUG 19:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638873" 19:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638873" 19:47:53 INFO [http_cmd]: http status code OK 19:47:53 DEBUG --- status code --- 19:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:53 DEBUG --- http response --- 19:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:53 DEBUG ----- output ----- 19:47:53 DEBUG { 19:47:53 DEBUG "status": "success", 19:47:53 DEBUG "data": { 19:47:53 DEBUG "resultType": "vector", 19:47:53 DEBUG "result": [ 19:47:53 DEBUG { 19:47:53 DEBUG "metric": { 19:47:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:53 DEBUG }, 19:47:53 DEBUG "value": [ 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG [...] 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:53 DEBUG }, 19:47:53 DEBUG "value": [ 19:47:53 DEBUG 1713638873, 19:47:53 DEBUG "2668.3" 19:47:53 DEBUG ] 19:47:53 DEBUG } 19:47:53 DEBUG ] 19:47:53 DEBUG } 19:47:53 DEBUG } 19:47:53 INFO [http_cmd]: http status code OK 19:47:53 DEBUG --- status code --- 19:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:53 DEBUG --- http response --- 19:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:47:53 DEBUG ----- output ----- 19:47:53 DEBUG { 19:47:53 DEBUG "status": "success", 19:47:53 DEBUG "data": { 19:47:53 DEBUG "resultType": "vector", 19:47:53 DEBUG "result": [ 19:47:53 DEBUG { 19:47:53 DEBUG "metric": { 19:47:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:47:53 DEBUG }, 19:47:53 DEBUG "value": [ 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG [...] 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:53 DEBUG }, 19:47:53 DEBUG "value": [ 19:47:53 DEBUG 1713638873, 19:47:53 DEBUG "1278.7208023641085" 19:47:53 DEBUG ] 19:47:53 DEBUG } 19:47:53 DEBUG ] 19:47:53 DEBUG } 19:47:53 DEBUG } 19:47:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638873" 19:47:53 INFO [http_cmd]: http status code OK 19:47:53 DEBUG --- status code --- 19:47:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:53 DEBUG --- http response --- 19:47:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:53 DEBUG ----- output ----- 19:47:53 DEBUG { 19:47:53 DEBUG "status": "success", 19:47:53 DEBUG "data": { 19:47:53 DEBUG "resultType": "vector", 19:47:53 DEBUG "result": [ 19:47:53 DEBUG { 19:47:53 DEBUG "metric": { 19:47:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:47:53 DEBUG }, 19:47:53 DEBUG "value": [ 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG [...] 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG 19:47:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:47:53 DEBUG }, 19:47:53 DEBUG "value": [ 19:47:53 DEBUG 1713638873, 19:47:53 DEBUG "8.883599391903301" 19:47:53 DEBUG ] 19:47:53 DEBUG } 19:47:53 DEBUG ] 19:47:53 DEBUG } 19:47:53 DEBUG } 19:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638874" 19:47:54 INFO [http_cmd]: http status code OK 19:47:54 DEBUG --- status code --- 19:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:54 DEBUG --- http response --- 19:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:54 DEBUG ----- output ----- 19:47:54 DEBUG { 19:47:54 DEBUG "status": "success", 19:47:54 DEBUG "data": { 19:47:54 DEBUG "resultType": "vector", 19:47:54 DEBUG "result": [ 19:47:54 DEBUG { 19:47:54 DEBUG "metric": { 19:47:54 DEBUG "instance": "10.142.0.107:9100" 19:47:54 DEBUG }, 19:47:54 DEBUG "value": [ 19:47:54 DEBUG 19:47:54 DEBUG 19:47:54 DEBUG 19:47:54 DEBUG [...] 19:47:54 DEBUG 19:47:54 DEBUG 19:47:54 DEBUG 19:47:54 DEBUG "instance": "10.142.0.43:9100" 19:47:54 DEBUG }, 19:47:54 DEBUG "value": [ 19:47:54 DEBUG 1713638874, 19:47:54 DEBUG "3598919908352" 19:47:54 DEBUG ] 19:47:54 DEBUG } 19:47:54 DEBUG ] 19:47:54 DEBUG } 19:47:54 DEBUG } 19:47:54 INFO 19:47:54 INFO [loop_until]: kubectl --namespace=xlou top node 19:47:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:47:54 INFO [loop_until]: OK (rc = 0) 19:47:54 DEBUG --- stdout --- 19:47:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6204Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7525m 25% 7642Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16375m 54% 11373Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3340Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18969m 63% 13480Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 146m 0% 5215Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17223m 57% 11946Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14443m 48% 2964Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19401m 64% 10888Mi 9% gke-xlou-cdm-ds-0f601751-j488 7842m 26% 7184Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17242m 57% 15270Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8109m 27% 7841Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18972m 63% 12938Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8787m 55% 6370Mi 10% 19:47:54 DEBUG --- stderr --- 19:47:54 DEBUG 19:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638874" 19:47:54 INFO [http_cmd]: http status code OK 19:47:54 DEBUG --- status code --- 19:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:54 DEBUG --- http response --- 19:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:54 DEBUG ----- output ----- 19:47:54 DEBUG { 19:47:54 DEBUG "status": "success", 19:47:54 DEBUG "data": { 19:47:54 DEBUG "resultType": "vector", 19:47:54 DEBUG "result": [ 19:47:54 DEBUG { 19:47:54 DEBUG "metric": { 19:47:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:54 DEBUG }, 19:47:54 DEBUG "value": [ 19:47:54 DEBUG 19:47:54 DEBUG 19:47:54 DEBUG 19:47:54 DEBUG [...] 19:47:54 DEBUG 19:47:54 DEBUG 19:47:54 DEBUG 19:47:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:54 DEBUG }, 19:47:54 DEBUG "value": [ 19:47:54 DEBUG 1713638874, 19:47:54 DEBUG "2668.6" 19:47:54 DEBUG ] 19:47:54 DEBUG } 19:47:54 DEBUG ] 19:47:54 DEBUG } 19:47:54 DEBUG } 19:47:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638874" 19:47:54 INFO [http_cmd]: http status code OK 19:47:54 DEBUG --- status code --- 19:47:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:54 DEBUG --- http response --- 19:47:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:54 DEBUG ----- output ----- 19:47:54 DEBUG { 19:47:54 DEBUG "status": "success", 19:47:54 DEBUG "data": { 19:47:54 DEBUG "resultType": "vector", 19:47:54 DEBUG "result": [ 19:47:54 DEBUG { 19:47:54 DEBUG "metric": { 19:47:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:54 DEBUG }, 19:47:54 DEBUG "value": [ 19:47:54 DEBUG 19:47:54 DEBUG 19:47:54 DEBUG 19:47:54 DEBUG [...] 19:47:54 DEBUG 19:47:54 DEBUG 19:47:54 DEBUG 19:47:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:54 DEBUG }, 19:47:54 DEBUG "value": [ 19:47:54 DEBUG 1713638874, 19:47:54 DEBUG "2668.3333333333335" 19:47:54 DEBUG ] 19:47:54 DEBUG } 19:47:54 DEBUG ] 19:47:54 DEBUG } 19:47:54 DEBUG } 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638875" 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 DEBUG --- status code --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG { 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG "data": { 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG "result": [ 19:47:55 DEBUG { 19:47:55 DEBUG "metric": { 19:47:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:47:55 DEBUG }, 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG [...] 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "pod": "ds-cts-0" 19:47:55 DEBUG }, 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG "0" 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638875" 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638875" 19:47:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638875" 19:47:55 DEBUG --- status code --- 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 DEBUG --- status code --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 DEBUG --- status code --- 19:47:55 DEBUG --- status code --- 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 DEBUG --- status code --- 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 DEBUG --- status code --- 19:47:55 DEBUG --- status code --- 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG --- http response --- 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 DEBUG --- status code --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG --- status code --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG --- status code --- 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG --- status code --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:47:55 INFO [http_cmd]: http status code OK 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG --- status code --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG --- status code --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG { 19:47:55 DEBUG --- status code --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG { 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG --- status code --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG { 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG "data": { 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG "data": { 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG --- http response --- 19:47:55 DEBUG { 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG { 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG { 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG "result": [ 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG "result": [ 19:47:55 DEBUG { 19:47:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:47:55 DEBUG { 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG { 19:47:55 DEBUG { 19:47:55 DEBUG "metric": { 19:47:55 DEBUG "pod": "ds-cts-0" 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG { 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG { 19:47:55 DEBUG "data": { 19:47:55 DEBUG { 19:47:55 DEBUG { 19:47:55 DEBUG "metric": { 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG "data": { 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG }, 19:47:55 DEBUG "value": [ 19:47:55 DEBUG "data": { 19:47:55 DEBUG "data": { 19:47:55 DEBUG ----- output ----- 19:47:55 DEBUG "data": { 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG "pod": "ds-cts-0" 19:47:55 DEBUG }, 19:47:55 DEBUG { 19:47:55 DEBUG "data": { 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG "data": { 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG { 19:47:55 DEBUG { 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG "data": { 19:47:55 DEBUG "result": [ 19:47:55 DEBUG "data": { 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG "value": [ 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG "data": { 19:47:55 DEBUG "result": [ 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG [...] 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG "result": [ 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG "result": [ 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG { 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG "data": { 19:47:55 DEBUG 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG { 19:47:55 DEBUG "result": [ 19:47:55 DEBUG 19:47:55 DEBUG { 19:47:55 DEBUG "status": "success", 19:47:55 DEBUG "result": [ 19:47:55 DEBUG "data": { 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG "result": [ 19:47:55 DEBUG "metric": { 19:47:55 DEBUG "result": [ 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG 19:47:55 DEBUG "result": [ 19:47:55 DEBUG "result": [ 19:47:55 DEBUG { 19:47:55 DEBUG "metric": { 19:47:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:47:55 DEBUG "metric": { 19:47:55 DEBUG { 19:47:55 DEBUG { 19:47:55 DEBUG "result": [ 19:47:55 DEBUG { 19:47:55 DEBUG "metric": { 19:47:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:47:55 DEBUG { 19:47:55 DEBUG "result": [ 19:47:55 DEBUG 19:47:55 DEBUG { 19:47:55 DEBUG }, 19:47:55 DEBUG "metric": { 19:47:55 DEBUG 19:47:55 DEBUG "instance": "10.142.0.107:9100" 19:47:55 DEBUG "metric": { 19:47:55 DEBUG "metric": { 19:47:55 DEBUG "data": { 19:47:55 DEBUG { 19:47:55 DEBUG "instance": "10.142.0.107:9100" 19:47:55 DEBUG }, 19:47:55 DEBUG "metric": { 19:47:55 DEBUG { 19:47:55 DEBUG "metric": { 19:47:55 DEBUG [...] 19:47:55 DEBUG "metric": { 19:47:55 DEBUG "value": [ 19:47:55 DEBUG { 19:47:55 DEBUG 19:47:55 DEBUG }, 19:47:55 DEBUG "instance": "10.142.0.107:9100" 19:47:55 DEBUG "instance": "10.142.0.107:9100" 19:47:55 DEBUG "resultType": "vector", 19:47:55 DEBUG "metric": { 19:47:55 DEBUG }, 19:47:55 DEBUG "value": [ 19:47:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:55 DEBUG "pod": "ds-idrepo-0" 19:47:55 DEBUG 19:47:55 DEBUG "pod": "ds-idrepo-0" 19:47:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "pod": "ds-cts-2" 19:47:55 DEBUG }, 19:47:55 DEBUG }, 19:47:55 DEBUG }, 19:47:55 DEBUG "result": [ 19:47:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 19:47:55 DEBUG }, 19:47:55 DEBUG }, 19:47:55 DEBUG 19:47:55 DEBUG }, 19:47:55 DEBUG }, 19:47:55 DEBUG 19:47:55 DEBUG "value": [ 19:47:55 DEBUG "value": [ 19:47:55 DEBUG "metric": { 19:47:55 DEBUG "value": [ 19:47:55 DEBUG "value": [ 19:47:55 DEBUG { 19:47:55 DEBUG "metric": { 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "value": [ 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 19:47:55 DEBUG "value": [ 19:47:55 DEBUG "value": [ 19:47:55 DEBUG [...] 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG 19:47:55 DEBUG "pod": "ds-cts-0" 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:47:55 DEBUG }, 19:47:55 DEBUG }, 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "pod": "ds-cts-2" 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "0.00010000000000000003" 19:47:55 DEBUG 19:47:55 DEBUG }, 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG [...] 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG }, 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG ] 19:47:55 DEBUG 19:47:55 DEBUG [...] 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG [...] 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 19:47:55 DEBUG [...] 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG [...] 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG [...] 19:47:55 DEBUG [...] 19:47:55 DEBUG 19:47:55 DEBUG [...] 19:47:55 DEBUG 19:47:55 DEBUG }, 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 19:47:55 DEBUG "instance": "10.142.0.43:9100" 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG } 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 19:47:55 DEBUG }, 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 19:47:55 DEBUG [...] 19:47:55 DEBUG ] 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG 19:47:55 DEBUG [...] 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG } 19:47:55 DEBUG "instance": "10.142.0.43:9100" 19:47:55 DEBUG [...] 19:47:55 DEBUG }, 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:55 DEBUG "0" 19:47:55 DEBUG ] 19:47:55 DEBUG "882835456" 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG "80899652.83333334" 19:47:55 DEBUG 19:47:55 DEBUG } 19:47:55 DEBUG }, 19:47:55 DEBUG "value": [ 19:47:55 DEBUG "value": [ 19:47:55 DEBUG "pod": "ds-cts-2" 19:47:55 DEBUG "pod": "ds-idrepo-2" 19:47:55 DEBUG }, 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG 19:47:55 DEBUG ] 19:47:55 DEBUG [...] 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG }, 19:47:55 DEBUG "value": [ 19:47:55 DEBUG ] 19:47:55 DEBUG }, 19:47:55 DEBUG } 19:47:55 DEBUG "instance": "10.142.0.43:9100" 19:47:55 DEBUG "instance": "10.142.0.43:9100" 19:47:55 DEBUG }, 19:47:55 DEBUG } 19:47:55 DEBUG }, 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG "pod": "ds-cts-2" 19:47:55 DEBUG "0" 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG 19:47:55 DEBUG "pod": "ds-cts-0" 19:47:55 DEBUG }, 19:47:55 DEBUG "value": [ 19:47:55 DEBUG }, 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 19:47:55 DEBUG }, 19:47:55 DEBUG "value": [ 19:47:55 DEBUG "value": [ 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG "0.10099999999998582" 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG "value": [ 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG "479621.5" 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG "0.2399333333333639" 19:47:55 DEBUG 19:47:55 DEBUG "0" 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG 19:47:55 DEBUG "0" 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG "0.155" 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG "80894967.86666667" 19:47:55 DEBUG ] 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG } 19:47:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:47:55 DEBUG ] 19:47:55 DEBUG ] 19:47:55 DEBUG ] 19:47:55 DEBUG "0" 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG "29.966666666666665" 19:47:55 DEBUG ] 19:47:55 DEBUG }, 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG "value": [ 19:47:55 DEBUG 1713638875, 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG "0" 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG ] 19:47:55 DEBUG ] 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG } 19:47:55 DEBUG } 19:48:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638931" 19:48:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638931" 19:48:51 ERROR [http_cmd]: ERROR 19:48:51 ERROR --- status code --- 19:48:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:48:51 ERROR --- http response --- 19:48:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:48:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:48:51 INFO [http_cmd]: http status code OK 19:48:51 DEBUG --- status code --- 19:48:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:51 DEBUG --- http response --- 19:48:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638933" 19:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638933" 19:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638933" 19:48:53 INFO [http_cmd]: http status code OK 19:48:53 DEBUG --- status code --- 19:48:53 INFO [http_cmd]: http status code OK 19:48:53 INFO [http_cmd]: http status code OK 19:48:53 DEBUG --- status code --- 19:48:53 DEBUG --- status code --- 19:48:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:53 DEBUG --- http response --- 19:48:53 DEBUG --- http response --- 19:48:53 DEBUG --- http response --- 19:48:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:53 DEBUG ----- output ----- 19:48:53 DEBUG ----- output ----- 19:48:53 DEBUG ----- output ----- 19:48:53 DEBUG { 19:48:53 DEBUG { 19:48:53 DEBUG { 19:48:53 DEBUG "status": "success", 19:48:53 DEBUG "status": "success", 19:48:53 DEBUG "status": "success", 19:48:53 DEBUG "data": { 19:48:53 DEBUG "data": { 19:48:53 DEBUG "data": { 19:48:53 DEBUG "resultType": "vector", 19:48:53 DEBUG "resultType": "vector", 19:48:53 DEBUG "resultType": "vector", 19:48:53 DEBUG "result": [ 19:48:53 DEBUG "result": [ 19:48:53 DEBUG "result": [ 19:48:53 DEBUG { 19:48:53 DEBUG { 19:48:53 DEBUG { 19:48:53 DEBUG "metric": { 19:48:53 DEBUG "metric": { 19:48:53 DEBUG "metric": { 19:48:53 DEBUG "pod": "ds-idrepo-0" 19:48:53 DEBUG "pod": "ds-idrepo-0" 19:48:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:53 DEBUG }, 19:48:53 DEBUG }, 19:48:53 DEBUG }, 19:48:53 DEBUG "value": [ 19:48:53 DEBUG "value": [ 19:48:53 DEBUG "value": [ 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG [...] 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG [...] 19:48:53 DEBUG "pod": "ds-idrepo-2" 19:48:53 DEBUG [...] 19:48:53 DEBUG 19:48:53 DEBUG }, 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG "value": [ 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 1713638933, 19:48:53 DEBUG 19:48:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:53 DEBUG "0" 19:48:53 DEBUG }, 19:48:53 DEBUG "pod": "ds-idrepo-2" 19:48:53 DEBUG ] 19:48:53 DEBUG "value": [ 19:48:53 DEBUG }, 19:48:53 DEBUG } 19:48:53 DEBUG 1713638933, 19:48:53 DEBUG "value": [ 19:48:53 DEBUG ] 19:48:53 DEBUG "2665.4666666666667" 19:48:53 DEBUG 1713638933, 19:48:53 DEBUG } 19:48:53 DEBUG ] 19:48:53 DEBUG "0" 19:48:53 DEBUG } 19:48:53 DEBUG } 19:48:53 DEBUG ] 19:48:53 DEBUG ] 19:48:53 DEBUG } 19:48:53 DEBUG } 19:48:53 DEBUG ] 19:48:53 DEBUG } 19:48:53 DEBUG } 19:48:53 DEBUG } 19:48:53 INFO 19:48:53 INFO [loop_until]: kubectl --namespace=xlou top pods 19:48:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:48:53 INFO [loop_until]: OK (rc = 0) 19:48:53 DEBUG --- stdout --- 19:48:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17310m 8652Mi am-767f6b67b5-qcqz2 17186m 9065Mi am-767f6b67b5-vbltz 18308m 8668Mi ds-cts-0 18040m 10276Mi ds-cts-1 17014m 10237Mi ds-cts-2 19024m 10259Mi ds-idrepo-0 7387m 3888Mi ds-idrepo-1 7340m 3882Mi ds-idrepo-2 7426m 3778Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 23m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8871m 749Mi 19:48:53 DEBUG --- stderr --- 19:48:53 DEBUG 19:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638933" 19:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638933" 19:48:53 INFO [http_cmd]: http status code OK 19:48:53 DEBUG --- status code --- 19:48:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:53 DEBUG --- http response --- 19:48:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:53 DEBUG ----- output ----- 19:48:53 DEBUG { 19:48:53 DEBUG "status": "success", 19:48:53 DEBUG "data": { 19:48:53 DEBUG "resultType": "vector", 19:48:53 DEBUG "result": [ 19:48:53 DEBUG { 19:48:53 DEBUG "metric": { 19:48:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:53 DEBUG }, 19:48:53 DEBUG "value": [ 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG [...] 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:53 DEBUG }, 19:48:53 DEBUG "value": [ 19:48:53 DEBUG 1713638933, 19:48:53 DEBUG "2666.233333333333" 19:48:53 DEBUG ] 19:48:53 DEBUG } 19:48:53 DEBUG ] 19:48:53 DEBUG } 19:48:53 DEBUG } 19:48:53 INFO [http_cmd]: http status code OK 19:48:53 DEBUG --- status code --- 19:48:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:53 DEBUG --- http response --- 19:48:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:48:53 DEBUG ----- output ----- 19:48:53 DEBUG { 19:48:53 DEBUG "status": "success", 19:48:53 DEBUG "data": { 19:48:53 DEBUG "resultType": "vector", 19:48:53 DEBUG "result": [ 19:48:53 DEBUG { 19:48:53 DEBUG "metric": { 19:48:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:48:53 DEBUG }, 19:48:53 DEBUG "value": [ 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG [...] 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:48:53 DEBUG }, 19:48:53 DEBUG "value": [ 19:48:53 DEBUG 1713638933, 19:48:53 DEBUG "0" 19:48:53 DEBUG ] 19:48:53 DEBUG } 19:48:53 DEBUG ] 19:48:53 DEBUG } 19:48:53 DEBUG } 19:48:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638933" 19:48:53 INFO [http_cmd]: http status code OK 19:48:53 DEBUG --- status code --- 19:48:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:53 DEBUG --- http response --- 19:48:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:53 DEBUG ----- output ----- 19:48:53 DEBUG { 19:48:53 DEBUG "status": "success", 19:48:53 DEBUG "data": { 19:48:53 DEBUG "resultType": "vector", 19:48:53 DEBUG "result": [ 19:48:53 DEBUG { 19:48:53 DEBUG "metric": { 19:48:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:48:53 DEBUG }, 19:48:53 DEBUG "value": [ 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG [...] 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG 19:48:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:48:53 DEBUG }, 19:48:53 DEBUG "value": [ 19:48:53 DEBUG 1713638933, 19:48:53 DEBUG "8.893477674741629" 19:48:53 DEBUG ] 19:48:53 DEBUG } 19:48:53 DEBUG ] 19:48:53 DEBUG } 19:48:53 DEBUG } 19:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638934" 19:48:54 INFO [http_cmd]: http status code OK 19:48:54 DEBUG --- status code --- 19:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:54 DEBUG --- http response --- 19:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:54 DEBUG ----- output ----- 19:48:54 DEBUG { 19:48:54 DEBUG "status": "success", 19:48:54 DEBUG "data": { 19:48:54 DEBUG "resultType": "vector", 19:48:54 DEBUG "result": [ 19:48:54 DEBUG { 19:48:54 DEBUG "metric": { 19:48:54 DEBUG "instance": "10.142.0.107:9100" 19:48:54 DEBUG }, 19:48:54 DEBUG "value": [ 19:48:54 DEBUG 19:48:54 DEBUG 19:48:54 DEBUG 19:48:54 DEBUG [...] 19:48:54 DEBUG 19:48:54 DEBUG 19:48:54 DEBUG 19:48:54 DEBUG "instance": "10.142.0.43:9100" 19:48:54 DEBUG }, 19:48:54 DEBUG "value": [ 19:48:54 DEBUG 1713638934, 19:48:54 DEBUG "3603311041536" 19:48:54 DEBUG ] 19:48:54 DEBUG } 19:48:54 DEBUG ] 19:48:54 DEBUG } 19:48:54 DEBUG } 19:48:54 INFO 19:48:54 INFO [loop_until]: kubectl --namespace=xlou top node 19:48:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:48:54 INFO [loop_until]: OK (rc = 0) 19:48:54 DEBUG --- stdout --- 19:48:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6217Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7549m 25% 7630Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16870m 56% 11375Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3343Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18471m 61% 13474Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5204Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17177m 57% 11953Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14541m 48% 2956Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19429m 65% 10889Mi 9% gke-xlou-cdm-ds-0f601751-j488 8138m 27% 7179Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17120m 57% 15296Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8078m 27% 7851Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19443m 65% 12936Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9076m 57% 6370Mi 10% 19:48:54 DEBUG --- stderr --- 19:48:54 DEBUG 19:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638934" 19:48:54 INFO [http_cmd]: http status code OK 19:48:54 DEBUG --- status code --- 19:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:54 DEBUG --- http response --- 19:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:54 DEBUG ----- output ----- 19:48:54 DEBUG { 19:48:54 DEBUG "status": "success", 19:48:54 DEBUG "data": { 19:48:54 DEBUG "resultType": "vector", 19:48:54 DEBUG "result": [ 19:48:54 DEBUG { 19:48:54 DEBUG "metric": { 19:48:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:54 DEBUG }, 19:48:54 DEBUG "value": [ 19:48:54 DEBUG 19:48:54 DEBUG 19:48:54 DEBUG 19:48:54 DEBUG [...] 19:48:54 DEBUG 19:48:54 DEBUG 19:48:54 DEBUG 19:48:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:54 DEBUG }, 19:48:54 DEBUG "value": [ 19:48:54 DEBUG 1713638934, 19:48:54 DEBUG "2666.2666666666664" 19:48:54 DEBUG ] 19:48:54 DEBUG } 19:48:54 DEBUG ] 19:48:54 DEBUG } 19:48:54 DEBUG } 19:48:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638934" 19:48:54 INFO [http_cmd]: http status code OK 19:48:54 DEBUG --- status code --- 19:48:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:54 DEBUG --- http response --- 19:48:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:54 DEBUG ----- output ----- 19:48:54 DEBUG { 19:48:54 DEBUG "status": "success", 19:48:54 DEBUG "data": { 19:48:54 DEBUG "resultType": "vector", 19:48:54 DEBUG "result": [ 19:48:54 DEBUG { 19:48:54 DEBUG "metric": { 19:48:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:54 DEBUG }, 19:48:54 DEBUG "value": [ 19:48:54 DEBUG 19:48:54 DEBUG 19:48:54 DEBUG 19:48:54 DEBUG [...] 19:48:54 DEBUG 19:48:54 DEBUG 19:48:54 DEBUG 19:48:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:54 DEBUG }, 19:48:54 DEBUG "value": [ 19:48:54 DEBUG 1713638934, 19:48:54 DEBUG "2665.5333333333333" 19:48:54 DEBUG ] 19:48:54 DEBUG } 19:48:54 DEBUG ] 19:48:54 DEBUG } 19:48:54 DEBUG } 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638935" 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 DEBUG --- status code --- 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG { 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG "data": { 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "result": [ 19:48:55 DEBUG { 19:48:55 DEBUG "metric": { 19:48:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:48:55 DEBUG }, 19:48:55 DEBUG "value": [ 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG [...] 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG "pod": "idm-b85f9d978-txpwv" 19:48:55 DEBUG }, 19:48:55 DEBUG "value": [ 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG "0.002197629057562301" 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638935" 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638935" 19:48:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638935" 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 DEBUG --- status code --- 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 DEBUG --- status code --- 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 DEBUG --- status code --- 19:48:55 DEBUG --- status code --- 19:48:55 DEBUG --- status code --- 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 DEBUG --- status code --- 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG --- status code --- 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG --- status code --- 19:48:55 DEBUG --- status code --- 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 DEBUG --- status code --- 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG --- status code --- 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG --- status code --- 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG --- status code --- 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 INFO [http_cmd]: http status code OK 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG --- status code --- 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:48:55 DEBUG --- status code --- 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:55 DEBUG { 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG { 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG { 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG { 19:48:55 DEBUG --- http response --- 19:48:55 DEBUG { 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG { 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG { 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:48:55 DEBUG { 19:48:55 DEBUG { 19:48:55 DEBUG { 19:48:55 DEBUG { 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG { 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG { 19:48:55 DEBUG "data": { 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG "data": { 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG "data": { 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG "data": { 19:48:55 DEBUG ----- output ----- 19:48:55 DEBUG "data": { 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "data": { 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "result": [ 19:48:55 DEBUG { 19:48:55 DEBUG "data": { 19:48:55 DEBUG "data": { 19:48:55 DEBUG "data": { 19:48:55 DEBUG "data": { 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "data": { 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG { 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG "data": { 19:48:55 DEBUG "result": [ 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG { 19:48:55 DEBUG "status": "success", 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "result": [ 19:48:55 DEBUG { 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "result": [ 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "result": [ 19:48:55 DEBUG "data": { 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG { 19:48:55 DEBUG "result": [ 19:48:55 DEBUG "metric": { 19:48:55 DEBUG "data": { 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "metric": { 19:48:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:55 DEBUG "result": [ 19:48:55 DEBUG "result": [ 19:48:55 DEBUG { 19:48:55 DEBUG "result": [ 19:48:55 DEBUG { 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "result": [ 19:48:55 DEBUG "result": [ 19:48:55 DEBUG "metric": { 19:48:55 DEBUG { 19:48:55 DEBUG "instance": "10.142.0.107:9100" 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "result": [ 19:48:55 DEBUG }, 19:48:55 DEBUG "data": { 19:48:55 DEBUG { 19:48:55 DEBUG { 19:48:55 DEBUG "metric": { 19:48:55 DEBUG { 19:48:55 DEBUG "metric": { 19:48:55 DEBUG "result": [ 19:48:55 DEBUG { 19:48:55 DEBUG { 19:48:55 DEBUG "pod": "ds-idrepo-0" 19:48:55 DEBUG "metric": { 19:48:55 DEBUG }, 19:48:55 DEBUG "result": [ 19:48:55 DEBUG { 19:48:55 DEBUG "value": [ 19:48:55 DEBUG "resultType": "vector", 19:48:55 DEBUG "metric": { 19:48:55 DEBUG "metric": { 19:48:55 DEBUG "pod": "ds-cts-0" 19:48:55 DEBUG "metric": { 19:48:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:48:55 DEBUG { 19:48:55 DEBUG "metric": { 19:48:55 DEBUG "metric": { 19:48:55 DEBUG }, 19:48:55 DEBUG "pod": "ds-cts-0" 19:48:55 DEBUG "value": [ 19:48:55 DEBUG { 19:48:55 DEBUG "metric": { 19:48:55 DEBUG 19:48:55 DEBUG "result": [ 19:48:55 DEBUG "instance": "10.142.0.107:9100" 19:48:55 DEBUG "pod": "ds-cts-0" 19:48:55 DEBUG }, 19:48:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:55 DEBUG }, 19:48:55 DEBUG "metric": { 19:48:55 DEBUG "pod": "ds-idrepo-0" 19:48:55 DEBUG "instance": "10.142.0.107:9100" 19:48:55 DEBUG "value": [ 19:48:55 DEBUG }, 19:48:55 DEBUG 19:48:55 DEBUG "metric": { 19:48:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:48:55 DEBUG 19:48:55 DEBUG { 19:48:55 DEBUG }, 19:48:55 DEBUG }, 19:48:55 DEBUG "value": [ 19:48:55 DEBUG }, 19:48:55 DEBUG "value": [ 19:48:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:48:55 DEBUG }, 19:48:55 DEBUG }, 19:48:55 DEBUG 19:48:55 DEBUG "value": [ 19:48:55 DEBUG 19:48:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:48:55 DEBUG }, 19:48:55 DEBUG 19:48:55 DEBUG "metric": { 19:48:55 DEBUG "value": [ 19:48:55 DEBUG "value": [ 19:48:55 DEBUG 19:48:55 DEBUG "value": [ 19:48:55 DEBUG 19:48:55 DEBUG }, 19:48:55 DEBUG "value": [ 19:48:55 DEBUG "value": [ 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG }, 19:48:55 DEBUG "value": [ 19:48:55 DEBUG [...] 19:48:55 DEBUG "instance": "10.142.0.107:9100" 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG "value": [ 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG [...] 19:48:55 DEBUG "value": [ 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG }, 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG [...] 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG "value": [ 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG [...] 19:48:55 DEBUG 19:48:55 DEBUG [...] 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG [...] 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG [...] 19:48:55 DEBUG [...] 19:48:55 DEBUG 19:48:55 DEBUG [...] 19:48:55 DEBUG 19:48:55 DEBUG [...] 19:48:55 DEBUG 19:48:55 DEBUG [...] 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG [...] 19:48:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG "instance": "10.142.0.43:9100" 19:48:55 DEBUG [...] 19:48:55 DEBUG 19:48:55 DEBUG }, 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG "pod": "ds-cts-2" 19:48:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:55 DEBUG 19:48:55 DEBUG [...] 19:48:55 DEBUG 19:48:55 DEBUG "pod": "ds-cts-2" 19:48:55 DEBUG 19:48:55 DEBUG }, 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG "value": [ 19:48:55 DEBUG [...] 19:48:55 DEBUG 19:48:55 DEBUG }, 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG }, 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG }, 19:48:55 DEBUG "pod": "ds-cts-2" 19:48:55 DEBUG "value": [ 19:48:55 DEBUG 19:48:55 DEBUG 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG 19:48:55 DEBUG "instance": "10.142.0.43:9100" 19:48:55 DEBUG "value": [ 19:48:55 DEBUG "pod": "ds-cts-2" 19:48:55 DEBUG 19:48:55 DEBUG "value": [ 19:48:55 DEBUG "pod": "ds-idrepo-2" 19:48:55 DEBUG 19:48:55 DEBUG "instance": "10.142.0.43:9100" 19:48:55 DEBUG "value": [ 19:48:55 DEBUG }, 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG 19:48:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:48:55 DEBUG "479402" 19:48:55 DEBUG 19:48:55 DEBUG }, 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG }, 19:48:55 DEBUG "pod": "idm-b85f9d978-txpwv" 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG }, 19:48:55 DEBUG 19:48:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG "value": [ 19:48:55 DEBUG "0.08466666666665977" 19:48:55 DEBUG "pod": "idm-b85f9d978-txpwv" 19:48:55 DEBUG }, 19:48:55 DEBUG ] 19:48:55 DEBUG 19:48:55 DEBUG "value": [ 19:48:55 DEBUG "30" 19:48:55 DEBUG "value": [ 19:48:55 DEBUG }, 19:48:55 DEBUG "0.155" 19:48:55 DEBUG "value": [ 19:48:55 DEBUG }, 19:48:55 DEBUG "0" 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG }, 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG "value": [ 19:48:55 DEBUG } 19:48:55 DEBUG "instance": "10.142.0.43:9100" 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG ] 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG "value": [ 19:48:55 DEBUG ] 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG "0" 19:48:55 DEBUG ] 19:48:55 DEBUG "value": [ 19:48:55 DEBUG "value": [ 19:48:55 DEBUG }, 19:48:55 DEBUG ] 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG ] 19:48:55 DEBUG }, 19:48:55 DEBUG "80800989" 19:48:55 DEBUG } 19:48:55 DEBUG "0" 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG } 19:48:55 DEBUG "0.0006666666666666666" 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG "value": [ 19:48:55 DEBUG } 19:48:55 DEBUG "885288960" 19:48:55 DEBUG } 19:48:55 DEBUG "value": [ 19:48:55 DEBUG ] 19:48:55 DEBUG ] 19:48:55 DEBUG ] 19:48:55 DEBUG "0" 19:48:55 DEBUG ] 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG ] 19:48:55 DEBUG "0" 19:48:55 DEBUG "80805545.4" 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG } 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG 1713638935, 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG ] 19:48:55 DEBUG ] 19:48:55 DEBUG "48027705.489749864" 19:48:55 DEBUG } 19:48:55 DEBUG "0.227533333333263" 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG ] 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG ] 19:48:55 DEBUG ] 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG ] 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG ] 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:48:55 DEBUG } 19:49:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713638991" 19:49:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638991" 19:49:51 ERROR [http_cmd]: ERROR 19:49:51 ERROR --- status code --- 19:49:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:49:51 INFO [http_cmd]: http status code OK 19:49:51 ERROR --- http response --- 19:49:51 DEBUG --- status code --- 19:49:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:49:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:49:51 DEBUG --- http response --- 19:49:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:49:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638993" 19:49:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638993" 19:49:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638993" 19:49:53 INFO [http_cmd]: http status code OK 19:49:53 DEBUG --- status code --- 19:49:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:53 DEBUG --- http response --- 19:49:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:53 DEBUG ----- output ----- 19:49:53 DEBUG { 19:49:53 DEBUG "status": "success", 19:49:53 DEBUG "data": { 19:49:53 DEBUG "resultType": "vector", 19:49:53 DEBUG "result": [ 19:49:53 DEBUG { 19:49:53 DEBUG "metric": { 19:49:53 DEBUG "pod": "ds-idrepo-0" 19:49:53 DEBUG }, 19:49:53 DEBUG "value": [ 19:49:53 DEBUG 19:49:53 DEBUG 19:49:53 DEBUG 19:49:53 DEBUG [...] 19:49:53 DEBUG 19:49:53 DEBUG 19:49:53 DEBUG 19:49:53 DEBUG "pod": "ds-idrepo-2" 19:49:53 INFO [http_cmd]: http status code OK 19:49:53 DEBUG --- status code --- 19:49:53 INFO [http_cmd]: http status code OK 19:49:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:53 DEBUG }, 19:49:53 DEBUG --- status code --- 19:49:53 DEBUG --- http response --- 19:49:53 DEBUG "value": [ 19:49:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:53 DEBUG 1713638993, 19:49:53 DEBUG --- http response --- 19:49:53 DEBUG ----- output ----- 19:49:53 DEBUG "0" 19:49:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:53 DEBUG { 19:49:53 DEBUG ] 19:49:53 DEBUG ----- output ----- 19:49:53 DEBUG "status": "success", 19:49:53 DEBUG } 19:49:53 DEBUG { 19:49:53 DEBUG "data": { 19:49:53 DEBUG ] 19:49:53 DEBUG "status": "success", 19:49:53 DEBUG "resultType": "vector", 19:49:53 DEBUG } 19:49:53 DEBUG "data": { 19:49:53 DEBUG "result": [ 19:49:53 DEBUG } 19:49:53 DEBUG "resultType": "vector", 19:49:53 DEBUG { 19:49:53 DEBUG "result": [ 19:49:53 DEBUG "metric": { 19:49:53 DEBUG { 19:49:53 DEBUG "pod": "ds-idrepo-0" 19:49:53 DEBUG "metric": { 19:49:53 DEBUG }, 19:49:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:53 DEBUG "value": [ 19:49:53 DEBUG }, 19:49:53 DEBUG 19:49:53 DEBUG "value": [ 19:49:53 DEBUG 19:49:53 DEBUG 19:49:53 DEBUG 19:49:53 DEBUG 19:49:53 DEBUG [...] 19:49:53 DEBUG 19:49:53 DEBUG 19:49:53 DEBUG [...] 19:49:53 DEBUG 19:49:53 DEBUG 19:49:53 DEBUG 19:49:53 DEBUG 19:49:53 DEBUG "pod": "ds-idrepo-2" 19:49:53 DEBUG 19:49:53 DEBUG }, 19:49:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:53 DEBUG "value": [ 19:49:53 DEBUG }, 19:49:53 DEBUG 1713638993, 19:49:53 DEBUG "value": [ 19:49:53 DEBUG "0" 19:49:53 DEBUG 1713638993, 19:49:53 DEBUG ] 19:49:53 DEBUG "2659.366666666667" 19:49:53 DEBUG } 19:49:53 DEBUG ] 19:49:53 DEBUG ] 19:49:53 DEBUG } 19:49:53 DEBUG } 19:49:53 DEBUG ] 19:49:53 DEBUG } 19:49:53 DEBUG } 19:49:53 DEBUG } 19:49:53 INFO 19:49:53 INFO [loop_until]: kubectl --namespace=xlou top pods 19:49:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:49:53 INFO [loop_until]: OK (rc = 0) 19:49:53 DEBUG --- stdout --- 19:49:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17431m 8652Mi am-767f6b67b5-qcqz2 17575m 9063Mi am-767f6b67b5-vbltz 18286m 8664Mi ds-cts-0 18749m 10265Mi ds-cts-1 16567m 10238Mi ds-cts-2 18763m 10268Mi ds-idrepo-0 7415m 3888Mi ds-idrepo-1 7425m 3883Mi ds-idrepo-2 7394m 3778Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 13m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8740m 750Mi 19:49:53 DEBUG --- stderr --- 19:49:53 DEBUG 19:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638994" 19:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638994" 19:49:54 INFO [http_cmd]: http status code OK 19:49:54 INFO [http_cmd]: http status code OK 19:49:54 DEBUG --- status code --- 19:49:54 DEBUG --- status code --- 19:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:54 DEBUG --- http response --- 19:49:54 DEBUG --- http response --- 19:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713638994" 19:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:54 DEBUG ----- output ----- 19:49:54 DEBUG ----- output ----- 19:49:54 DEBUG { 19:49:54 DEBUG { 19:49:54 DEBUG "status": "success", 19:49:54 DEBUG "status": "success", 19:49:54 DEBUG "data": { 19:49:54 DEBUG "data": { 19:49:54 DEBUG "resultType": "vector", 19:49:54 DEBUG "resultType": "vector", 19:49:54 DEBUG "result": [ 19:49:54 DEBUG "result": [ 19:49:54 DEBUG { 19:49:54 DEBUG { 19:49:54 DEBUG "metric": { 19:49:54 DEBUG "metric": { 19:49:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:49:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:54 DEBUG }, 19:49:54 DEBUG }, 19:49:54 DEBUG "value": [ 19:49:54 DEBUG "value": [ 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG [...] 19:49:54 DEBUG [...] 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG "pod": "ds-cts-0" 19:49:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:54 DEBUG }, 19:49:54 DEBUG }, 19:49:54 DEBUG "value": [ 19:49:54 DEBUG "value": [ 19:49:54 DEBUG 1713638994, 19:49:54 DEBUG 1713638994, 19:49:54 DEBUG "40033029.17452727" 19:49:54 DEBUG "2663.9" 19:49:54 DEBUG ] 19:49:54 DEBUG ] 19:49:54 DEBUG } 19:49:54 DEBUG } 19:49:54 DEBUG ] 19:49:54 DEBUG ] 19:49:54 DEBUG } 19:49:54 DEBUG } 19:49:54 DEBUG } 19:49:54 DEBUG } 19:49:54 INFO [http_cmd]: http status code OK 19:49:54 DEBUG --- status code --- 19:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:54 DEBUG --- http response --- 19:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:54 DEBUG ----- output ----- 19:49:54 DEBUG { 19:49:54 DEBUG "status": "success", 19:49:54 DEBUG "data": { 19:49:54 DEBUG "resultType": "vector", 19:49:54 DEBUG "result": [ 19:49:54 DEBUG { 19:49:54 DEBUG "metric": { 19:49:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:49:54 DEBUG }, 19:49:54 DEBUG "value": [ 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG [...] 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:49:54 DEBUG }, 19:49:54 DEBUG "value": [ 19:49:54 DEBUG 1713638994, 19:49:54 DEBUG "8.89849009393611" 19:49:54 DEBUG ] 19:49:54 DEBUG } 19:49:54 DEBUG ] 19:49:54 DEBUG } 19:49:54 DEBUG } 19:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638994" 19:49:54 INFO [http_cmd]: http status code OK 19:49:54 DEBUG --- status code --- 19:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:54 DEBUG --- http response --- 19:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:54 DEBUG ----- output ----- 19:49:54 DEBUG { 19:49:54 DEBUG "status": "success", 19:49:54 DEBUG "data": { 19:49:54 DEBUG "resultType": "vector", 19:49:54 DEBUG "result": [ 19:49:54 DEBUG { 19:49:54 DEBUG "metric": { 19:49:54 DEBUG "instance": "10.142.0.107:9100" 19:49:54 DEBUG }, 19:49:54 DEBUG "value": [ 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG [...] 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG "instance": "10.142.0.43:9100" 19:49:54 DEBUG }, 19:49:54 DEBUG "value": [ 19:49:54 DEBUG 1713638994, 19:49:54 DEBUG "3606990132224" 19:49:54 DEBUG ] 19:49:54 DEBUG } 19:49:54 DEBUG ] 19:49:54 DEBUG } 19:49:54 DEBUG } 19:49:54 INFO 19:49:54 INFO [loop_until]: kubectl --namespace=xlou top node 19:49:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:49:54 INFO [loop_until]: OK (rc = 0) 19:49:54 DEBUG --- stdout --- 19:49:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6223Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7630m 25% 7633Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17017m 56% 11366Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 92m 0% 3330Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19150m 64% 13479Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5223Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17243m 57% 11950Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14575m 48% 2976Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19531m 65% 10888Mi 9% gke-xlou-cdm-ds-0f601751-j488 8148m 27% 7180Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16813m 56% 15309Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8082m 27% 7854Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18863m 63% 12927Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9106m 57% 6371Mi 10% 19:49:54 DEBUG --- stderr --- 19:49:54 DEBUG 19:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638994" 19:49:54 INFO [http_cmd]: http status code OK 19:49:54 DEBUG --- status code --- 19:49:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:54 DEBUG --- http response --- 19:49:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:54 DEBUG ----- output ----- 19:49:54 DEBUG { 19:49:54 DEBUG "status": "success", 19:49:54 DEBUG "data": { 19:49:54 DEBUG "resultType": "vector", 19:49:54 DEBUG "result": [ 19:49:54 DEBUG { 19:49:54 DEBUG "metric": { 19:49:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:54 DEBUG }, 19:49:54 DEBUG "value": [ 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG [...] 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG 19:49:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:54 DEBUG }, 19:49:54 DEBUG "value": [ 19:49:54 DEBUG 1713638994, 19:49:54 DEBUG "2664.733333333333" 19:49:54 DEBUG ] 19:49:54 DEBUG } 19:49:54 DEBUG ] 19:49:54 DEBUG } 19:49:54 DEBUG } 19:49:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638994" 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG { 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG "data": { 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "result": [ 19:49:55 DEBUG { 19:49:55 DEBUG "metric": { 19:49:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:55 DEBUG }, 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:55 DEBUG }, 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 1713638994, 19:49:55 DEBUG "2664.6666666666665" 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638995" 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG { 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG "data": { 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "result": [ 19:49:55 DEBUG { 19:49:55 DEBUG "metric": { 19:49:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:49:55 DEBUG }, 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG "pod": "ds-idrepo-2" 19:49:55 DEBUG }, 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG "7.412961847098305" 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713638995" 19:49:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713638995" 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 DEBUG --- status code --- 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG --- status code --- 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG --- status code --- 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 DEBUG --- status code --- 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG --- http response --- 19:49:55 INFO [http_cmd]: http status code OK 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:55 DEBUG --- status code --- 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG { 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG { 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG { 19:49:55 DEBUG --- http response --- 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:49:55 DEBUG { 19:49:55 DEBUG { 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:49:55 DEBUG { 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG { 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:49:55 DEBUG { 19:49:55 DEBUG { 19:49:55 DEBUG { 19:49:55 DEBUG { 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG "data": { 19:49:55 DEBUG "data": { 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG { 19:49:55 DEBUG "data": { 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG ----- output ----- 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG { 19:49:55 DEBUG "data": { 19:49:55 DEBUG "data": { 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG { 19:49:55 DEBUG "data": { 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "data": { 19:49:55 DEBUG { 19:49:55 DEBUG "data": { 19:49:55 DEBUG "data": { 19:49:55 DEBUG "data": { 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "data": { 19:49:55 DEBUG "result": [ 19:49:55 DEBUG "result": [ 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "data": { 19:49:55 DEBUG "result": [ 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "status": "success", 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "result": [ 19:49:55 DEBUG "data": { 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "result": [ 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG { 19:49:55 DEBUG { 19:49:55 DEBUG "data": { 19:49:55 DEBUG "result": [ 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG { 19:49:55 DEBUG "result": [ 19:49:55 DEBUG "data": { 19:49:55 DEBUG "result": [ 19:49:55 DEBUG { 19:49:55 DEBUG "result": [ 19:49:55 DEBUG "result": [ 19:49:55 DEBUG { 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "result": [ 19:49:55 DEBUG "metric": { 19:49:55 DEBUG "metric": { 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG "result": [ 19:49:55 DEBUG { 19:49:55 DEBUG "metric": { 19:49:55 DEBUG { 19:49:55 DEBUG "resultType": "vector", 19:49:55 DEBUG { 19:49:55 DEBUG "metric": { 19:49:55 DEBUG { 19:49:55 DEBUG "metric": { 19:49:55 DEBUG { 19:49:55 DEBUG "result": [ 19:49:55 DEBUG { 19:49:55 DEBUG "pod": "ds-cts-0" 19:49:55 DEBUG "instance": "10.142.0.107:9100" 19:49:55 DEBUG { 19:49:55 DEBUG "result": [ 19:49:55 DEBUG "metric": { 19:49:55 DEBUG "instance": "10.142.0.107:9100" 19:49:55 DEBUG "metric": { 19:49:55 DEBUG "result": [ 19:49:55 DEBUG { 19:49:55 DEBUG "pod": "ds-cts-0" 19:49:55 DEBUG "metric": { 19:49:55 DEBUG "instance": "10.142.0.107:9100" 19:49:55 DEBUG "metric": { 19:49:55 DEBUG { 19:49:55 DEBUG "metric": { 19:49:55 DEBUG }, 19:49:55 DEBUG }, 19:49:55 DEBUG "metric": { 19:49:55 DEBUG { 19:49:55 DEBUG "pod": "ds-idrepo-0" 19:49:55 DEBUG }, 19:49:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:55 DEBUG "metric": { 19:49:55 DEBUG "metric": { 19:49:55 DEBUG }, 19:49:55 DEBUG "instance": "10.142.0.107:9100" 19:49:55 DEBUG }, 19:49:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:49:55 DEBUG "metric": { 19:49:55 DEBUG "pod": "ds-idrepo-0" 19:49:55 DEBUG "value": [ 19:49:55 DEBUG "value": [ 19:49:55 DEBUG "pod": "ds-cts-0" 19:49:55 DEBUG "metric": { 19:49:55 DEBUG }, 19:49:55 DEBUG "value": [ 19:49:55 DEBUG }, 19:49:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:49:55 DEBUG "value": [ 19:49:55 DEBUG }, 19:49:55 DEBUG "value": [ 19:49:55 DEBUG }, 19:49:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:49:55 DEBUG }, 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG }, 19:49:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 19:49:55 DEBUG "value": [ 19:49:55 DEBUG }, 19:49:55 DEBUG }, 19:49:55 DEBUG 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 19:49:55 DEBUG "value": [ 19:49:55 DEBUG }, 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG "value": [ 19:49:55 DEBUG }, 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG "value": [ 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG "pod": "ds-cts-2" 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG }, 19:49:55 DEBUG 19:49:55 DEBUG "instance": "10.142.0.43:9100" 19:49:55 DEBUG [...] 19:49:55 DEBUG 19:49:55 DEBUG "instance": "10.142.0.43:9100" 19:49:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG "pod": "ds-cts-2" 19:49:55 DEBUG "instance": "10.142.0.43:9100" 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 19:49:55 DEBUG }, 19:49:55 DEBUG 19:49:55 DEBUG "pod": "ds-cts-2" 19:49:55 DEBUG }, 19:49:55 DEBUG }, 19:49:55 DEBUG 19:49:55 DEBUG 19:49:55 DEBUG }, 19:49:55 DEBUG }, 19:49:55 DEBUG "instance": "10.142.0.43:9100" 19:49:55 DEBUG }, 19:49:55 DEBUG 19:49:55 DEBUG "pod": "ds-idrepo-2" 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG "pod": "ds-cts-2" 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 19:49:55 DEBUG }, 19:49:55 DEBUG "value": [ 19:49:55 DEBUG "value": [ 19:49:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:55 DEBUG 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG "value": [ 19:49:55 DEBUG "value": [ 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG }, 19:49:55 DEBUG "29.966666666666665" 19:49:55 DEBUG }, 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG 19:49:55 DEBUG "pod": "ds-idrepo-2" 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG }, 19:49:55 DEBUG "pod": "ds-idrepo-2" 19:49:55 DEBUG "0.00023333333333333333" 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG "80796628.73333332" 19:49:55 DEBUG "0.06433333333334113" 19:49:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:49:55 DEBUG "value": [ 19:49:55 DEBUG ] 19:49:55 DEBUG "value": [ 19:49:55 DEBUG "80791722.30000001" 19:49:55 DEBUG "value": [ 19:49:55 DEBUG }, 19:49:55 DEBUG "value": [ 19:49:55 DEBUG "479983" 19:49:55 DEBUG "value": [ 19:49:55 DEBUG }, 19:49:55 DEBUG ] 19:49:55 DEBUG ] 19:49:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:49:55 DEBUG ] 19:49:55 DEBUG }, 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG } 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG ] 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG "0.17103333333326512" 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG ] 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG "value": [ 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG }, 19:49:55 DEBUG } 19:49:55 DEBUG "value": [ 19:49:55 DEBUG "0" 19:49:55 DEBUG ] 19:49:55 DEBUG "0" 19:49:55 DEBUG } 19:49:55 DEBUG "0" 19:49:55 DEBUG ] 19:49:55 DEBUG "0" 19:49:55 DEBUG } 19:49:55 DEBUG "0" 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG ] 19:49:55 DEBUG ] 19:49:55 DEBUG "value": [ 19:49:55 DEBUG ] 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG ] 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG ] 19:49:55 DEBUG ] 19:49:55 DEBUG "172636778.52348992" 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG 1713638995, 19:49:55 DEBUG } 19:49:55 DEBUG "886415360" 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG "0.1555" 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG ] 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:49:55 DEBUG } 19:50:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639052" 19:50:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639052" 19:50:52 INFO [http_cmd]: http status code OK 19:50:52 ERROR [http_cmd]: ERROR 19:50:52 DEBUG --- status code --- 19:50:52 ERROR --- status code --- 19:50:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:50:52 DEBUG --- http response --- 19:50:52 ERROR --- http response --- 19:50:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:50:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:50:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:50:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639053" 19:50:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639053" 19:50:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639053" 19:50:53 INFO [http_cmd]: http status code OK 19:50:53 DEBUG --- status code --- 19:50:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:53 DEBUG --- http response --- 19:50:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:53 DEBUG ----- output ----- 19:50:53 DEBUG { 19:50:53 DEBUG "status": "success", 19:50:53 DEBUG "data": { 19:50:53 DEBUG "resultType": "vector", 19:50:53 DEBUG "result": [ 19:50:53 DEBUG { 19:50:53 DEBUG "metric": { 19:50:53 DEBUG "pod": "ds-idrepo-0" 19:50:53 DEBUG }, 19:50:53 DEBUG "value": [ 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG [...] 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG "pod": "ds-idrepo-2" 19:50:53 DEBUG }, 19:50:53 DEBUG "value": [ 19:50:53 DEBUG 1713639053, 19:50:53 DEBUG "0" 19:50:53 DEBUG ] 19:50:53 DEBUG } 19:50:53 DEBUG ] 19:50:53 DEBUG } 19:50:53 DEBUG } 19:50:53 INFO [http_cmd]: http status code OK 19:50:53 INFO [http_cmd]: http status code OK 19:50:53 DEBUG --- status code --- 19:50:53 DEBUG --- status code --- 19:50:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:53 DEBUG --- http response --- 19:50:53 DEBUG --- http response --- 19:50:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:53 DEBUG ----- output ----- 19:50:53 DEBUG ----- output ----- 19:50:53 DEBUG { 19:50:53 DEBUG { 19:50:53 DEBUG "status": "success", 19:50:53 DEBUG "status": "success", 19:50:53 DEBUG "data": { 19:50:53 DEBUG "data": { 19:50:53 DEBUG "resultType": "vector", 19:50:53 DEBUG "resultType": "vector", 19:50:53 DEBUG "result": [ 19:50:53 DEBUG "result": [ 19:50:53 DEBUG { 19:50:53 DEBUG { 19:50:53 DEBUG "metric": { 19:50:53 DEBUG "metric": { 19:50:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:53 DEBUG "pod": "ds-idrepo-0" 19:50:53 DEBUG }, 19:50:53 DEBUG }, 19:50:53 DEBUG "value": [ 19:50:53 DEBUG "value": [ 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG [...] 19:50:53 DEBUG [...] 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG 19:50:53 DEBUG "pod": "ds-idrepo-2" 19:50:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:53 DEBUG }, 19:50:53 DEBUG }, 19:50:53 DEBUG "value": [ 19:50:53 DEBUG "value": [ 19:50:53 DEBUG 1713639053, 19:50:53 DEBUG "2671.9666666666667" 19:50:53 DEBUG ] 19:50:53 DEBUG 1713639053, 19:50:53 DEBUG } 19:50:53 DEBUG "0" 19:50:53 DEBUG ] 19:50:53 DEBUG ] 19:50:53 DEBUG } 19:50:53 DEBUG } 19:50:53 DEBUG } 19:50:53 DEBUG ] 19:50:53 DEBUG } 19:50:53 DEBUG } 19:50:53 INFO 19:50:53 INFO [loop_until]: kubectl --namespace=xlou top pods 19:50:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:50:53 INFO [loop_until]: OK (rc = 0) 19:50:53 DEBUG --- stdout --- 19:50:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16836m 8651Mi am-767f6b67b5-qcqz2 17061m 9065Mi am-767f6b67b5-vbltz 18105m 8665Mi ds-cts-0 18915m 10278Mi ds-cts-1 16644m 10245Mi ds-cts-2 19224m 10250Mi ds-idrepo-0 7391m 3888Mi ds-idrepo-1 7318m 3882Mi ds-idrepo-2 7499m 3778Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 16m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9056m 753Mi 19:50:53 DEBUG --- stderr --- 19:50:53 DEBUG 19:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639054" 19:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639054" 19:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639054" 19:50:54 INFO [http_cmd]: http status code OK 19:50:54 INFO [http_cmd]: http status code OK 19:50:54 DEBUG --- status code --- 19:50:54 DEBUG --- status code --- 19:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:54 DEBUG --- http response --- 19:50:54 DEBUG --- http response --- 19:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:50:54 DEBUG ----- output ----- 19:50:54 DEBUG ----- output ----- 19:50:54 DEBUG { 19:50:54 DEBUG { 19:50:54 DEBUG "status": "success", 19:50:54 DEBUG "status": "success", 19:50:54 DEBUG "data": { 19:50:54 DEBUG "resultType": "vector", 19:50:54 DEBUG "data": { 19:50:54 DEBUG "result": [ 19:50:54 DEBUG "resultType": "vector", 19:50:54 DEBUG { 19:50:54 DEBUG "result": [ 19:50:54 DEBUG "metric": { 19:50:54 DEBUG { 19:50:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:54 DEBUG "metric": { 19:50:54 DEBUG }, 19:50:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:50:54 DEBUG "value": [ 19:50:54 DEBUG }, 19:50:54 DEBUG 19:50:54 DEBUG "value": [ 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG [...] 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG [...] 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:54 DEBUG 19:50:54 DEBUG }, 19:50:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:50:54 DEBUG "value": [ 19:50:54 DEBUG }, 19:50:54 DEBUG 1713639054, 19:50:54 DEBUG "value": [ 19:50:54 DEBUG "2667.8333333333335" 19:50:54 DEBUG 1713639054, 19:50:54 DEBUG ] 19:50:54 DEBUG "0" 19:50:54 DEBUG } 19:50:54 DEBUG ] 19:50:54 DEBUG ] 19:50:54 DEBUG } 19:50:54 DEBUG } 19:50:54 DEBUG ] 19:50:54 DEBUG } 19:50:54 DEBUG } 19:50:54 DEBUG } 19:50:54 INFO [http_cmd]: http status code OK 19:50:54 DEBUG --- status code --- 19:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:54 DEBUG --- http response --- 19:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:54 DEBUG ----- output ----- 19:50:54 DEBUG { 19:50:54 DEBUG "status": "success", 19:50:54 DEBUG "data": { 19:50:54 DEBUG "resultType": "vector", 19:50:54 DEBUG "result": [ 19:50:54 DEBUG { 19:50:54 DEBUG "metric": { 19:50:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:50:54 DEBUG }, 19:50:54 DEBUG "value": [ 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG [...] 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:50:54 DEBUG }, 19:50:54 DEBUG "value": [ 19:50:54 DEBUG 1713639054, 19:50:54 DEBUG "8.941291475008795" 19:50:54 DEBUG ] 19:50:54 DEBUG } 19:50:54 DEBUG ] 19:50:54 DEBUG } 19:50:54 DEBUG } 19:50:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639054" 19:50:54 INFO [http_cmd]: http status code OK 19:50:54 DEBUG --- status code --- 19:50:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:54 DEBUG --- http response --- 19:50:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:54 DEBUG ----- output ----- 19:50:54 DEBUG { 19:50:54 DEBUG "status": "success", 19:50:54 DEBUG "data": { 19:50:54 DEBUG "resultType": "vector", 19:50:54 DEBUG "result": [ 19:50:54 DEBUG { 19:50:54 DEBUG "metric": { 19:50:54 DEBUG "instance": "10.142.0.107:9100" 19:50:54 DEBUG }, 19:50:54 DEBUG "value": [ 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG [...] 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG 19:50:54 DEBUG "instance": "10.142.0.43:9100" 19:50:54 DEBUG }, 19:50:54 DEBUG "value": [ 19:50:54 DEBUG 1713639054, 19:50:54 DEBUG "3610901588992" 19:50:54 DEBUG ] 19:50:54 DEBUG } 19:50:54 DEBUG ] 19:50:54 DEBUG } 19:50:54 DEBUG } 19:50:54 INFO 19:50:54 INFO [loop_until]: kubectl --namespace=xlou top node 19:50:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:50:54 INFO [loop_until]: OK (rc = 0) 19:50:54 DEBUG --- stdout --- 19:50:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6229Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7549m 25% 7634Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16946m 56% 11370Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 89m 0% 3339Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18972m 63% 13482Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5234Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17205m 57% 11951Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14553m 48% 2965Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19391m 64% 10886Mi 9% gke-xlou-cdm-ds-0f601751-j488 8207m 27% 7180Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16950m 56% 15309Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8120m 27% 7853Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19106m 63% 12942Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9131m 57% 6371Mi 10% 19:50:54 DEBUG --- stderr --- 19:50:54 DEBUG 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG { 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "data": { 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "result": [ 19:50:55 DEBUG { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG "2667.8" 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG { 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "data": { 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "result": [ 19:50:55 DEBUG { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG "2666.8" 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG { 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "data": { 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "result": [ 19:50:55 DEBUG { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG "3.964326651393166" 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG { 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "data": { 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "result": [ 19:50:55 DEBUG { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "pod": "ds-idrepo-0" 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG "pod": "ds-idrepo-2" 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG "0" 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639055" 19:50:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639055" 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 DEBUG --- status code --- 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 DEBUG --- status code --- 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 DEBUG --- status code --- 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 INFO [http_cmd]: http status code OK 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG --- status code --- 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG { 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:50:55 DEBUG --- http response --- 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG { 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG { 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG "data": { 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:50:55 DEBUG { 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG { 19:50:55 DEBUG { 19:50:55 DEBUG { 19:50:55 DEBUG { 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG { 19:50:55 DEBUG { 19:50:55 DEBUG { 19:50:55 DEBUG { 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG { 19:50:55 DEBUG ----- output ----- 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "data": { 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "data": { 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "result": [ 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG { 19:50:55 DEBUG "data": { 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "data": { 19:50:55 DEBUG "data": { 19:50:55 DEBUG "data": { 19:50:55 DEBUG "data": { 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "data": { 19:50:55 DEBUG "data": { 19:50:55 DEBUG "data": { 19:50:55 DEBUG "data": { 19:50:55 DEBUG { 19:50:55 DEBUG "data": { 19:50:55 DEBUG "status": "success", 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "result": [ 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "result": [ 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG "data": { 19:50:55 DEBUG "result": [ 19:50:55 DEBUG { 19:50:55 DEBUG "result": [ 19:50:55 DEBUG "result": [ 19:50:55 DEBUG "result": [ 19:50:55 DEBUG "result": [ 19:50:55 DEBUG { 19:50:55 DEBUG "result": [ 19:50:55 DEBUG "result": [ 19:50:55 DEBUG "result": [ 19:50:55 DEBUG "result": [ 19:50:55 DEBUG "pod": "ds-cts-0" 19:50:55 DEBUG "result": [ 19:50:55 DEBUG "resultType": "vector", 19:50:55 DEBUG { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG { 19:50:55 DEBUG { 19:50:55 DEBUG { 19:50:55 DEBUG { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG { 19:50:55 DEBUG { 19:50:55 DEBUG { 19:50:55 DEBUG { 19:50:55 DEBUG }, 19:50:55 DEBUG { 19:50:55 DEBUG "result": [ 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "instance": "10.142.0.107:9100" 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "instance": "10.142.0.107:9100" 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "value": [ 19:50:55 DEBUG "metric": { 19:50:55 DEBUG { 19:50:55 DEBUG "metric": { 19:50:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:55 DEBUG "pod": "ds-cts-0" 19:50:55 DEBUG "instance": "10.142.0.107:9100" 19:50:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:55 DEBUG "instance": "10.142.0.107:9100" 19:50:55 DEBUG }, 19:50:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:50:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:50:55 DEBUG }, 19:50:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:50:55 DEBUG 19:50:55 DEBUG "pod": "ds-cts-0" 19:50:55 DEBUG }, 19:50:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:50:55 DEBUG }, 19:50:55 DEBUG }, 19:50:55 DEBUG }, 19:50:55 DEBUG }, 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG }, 19:50:55 DEBUG 19:50:55 DEBUG "pod": "ds-idrepo-0" 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG "value": [ 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG "value": [ 19:50:55 DEBUG }, 19:50:55 DEBUG 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG "pod": "ds-cts-2" 19:50:55 DEBUG 19:50:55 DEBUG [...] 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG "instance": "10.142.0.43:9100" 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:50:55 DEBUG 19:50:55 DEBUG }, 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG }, 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG }, 19:50:55 DEBUG 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG "instance": "10.142.0.43:9100" 19:50:55 DEBUG 19:50:55 DEBUG }, 19:50:55 DEBUG 19:50:55 DEBUG 19:50:55 DEBUG "instance": "10.142.0.43:9100" 19:50:55 DEBUG "value": [ 19:50:55 DEBUG "instance": "10.142.0.43:9100" 19:50:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG "30.9" 19:50:55 DEBUG 19:50:55 DEBUG }, 19:50:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:55 DEBUG "value": [ 19:50:55 DEBUG "pod": "ds-cts-2" 19:50:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:50:55 DEBUG }, 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG }, 19:50:55 DEBUG }, 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:50:55 DEBUG "pod": "ds-cts-2" 19:50:55 DEBUG ] 19:50:55 DEBUG "pod": "ds-cts-2" 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG "479920" 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG "80854278.23333332" 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG }, 19:50:55 DEBUG }, 19:50:55 DEBUG } 19:50:55 DEBUG }, 19:50:55 DEBUG }, 19:50:55 DEBUG "80856809.3" 19:50:55 DEBUG ] 19:50:55 DEBUG }, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG ] 19:50:55 DEBUG "889327616" 19:50:55 DEBUG "0.23303333333342663" 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG "value": [ 19:50:55 DEBUG "value": [ 19:50:55 DEBUG ] 19:50:55 DEBUG "value": [ 19:50:55 DEBUG "value": [ 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG "value": [ 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG "0.0903333333333421" 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG ] 19:50:55 DEBUG "0" 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG } 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG 1713639055, 19:50:55 DEBUG "0" 19:50:55 DEBUG ] 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG "24491618.229671698" 19:50:55 DEBUG "0" 19:50:55 DEBUG } 19:50:55 DEBUG "0.0001333333333333333" 19:50:55 DEBUG "0.156" 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG "0" 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG ] 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:50:55 DEBUG } 19:51:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639112" 19:51:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639112" 19:51:52 INFO [http_cmd]: http status code OK 19:51:52 DEBUG --- status code --- 19:51:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:52 DEBUG --- http response --- 19:51:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:51:52 ERROR [http_cmd]: ERROR 19:51:52 ERROR --- status code --- 19:51:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:51:52 ERROR --- http response --- 19:51:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:51:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:51:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639113" 19:51:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639113" 19:51:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639113" 19:51:53 INFO [http_cmd]: http status code OK 19:51:53 DEBUG --- status code --- 19:51:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:53 DEBUG --- http response --- 19:51:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:53 DEBUG ----- output ----- 19:51:53 DEBUG { 19:51:53 DEBUG "status": "success", 19:51:53 DEBUG "data": { 19:51:53 DEBUG "resultType": "vector", 19:51:53 DEBUG "result": [ 19:51:53 DEBUG { 19:51:53 DEBUG "metric": { 19:51:53 DEBUG "pod": "ds-idrepo-0" 19:51:53 DEBUG }, 19:51:53 DEBUG "value": [ 19:51:53 DEBUG 19:51:53 DEBUG 19:51:53 DEBUG 19:51:53 DEBUG [...] 19:51:53 DEBUG 19:51:53 DEBUG 19:51:53 DEBUG 19:51:53 DEBUG "pod": "ds-idrepo-2" 19:51:53 DEBUG }, 19:51:53 DEBUG "value": [ 19:51:53 DEBUG 1713639113, 19:51:53 DEBUG "0" 19:51:53 DEBUG ] 19:51:53 DEBUG } 19:51:53 DEBUG ] 19:51:53 DEBUG } 19:51:53 DEBUG } 19:51:53 INFO [http_cmd]: http status code OK 19:51:53 DEBUG --- status code --- 19:51:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:53 DEBUG --- http response --- 19:51:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:53 DEBUG ----- output ----- 19:51:53 DEBUG { 19:51:53 DEBUG "status": "success", 19:51:53 DEBUG "data": { 19:51:53 DEBUG "resultType": "vector", 19:51:53 DEBUG "result": [ 19:51:53 DEBUG { 19:51:53 DEBUG "metric": { 19:51:53 DEBUG "pod": "ds-idrepo-0" 19:51:53 DEBUG }, 19:51:53 DEBUG "value": [ 19:51:53 DEBUG 19:51:53 DEBUG 19:51:53 DEBUG 19:51:53 DEBUG [...] 19:51:53 DEBUG 19:51:53 DEBUG 19:51:53 DEBUG 19:51:53 DEBUG "pod": "ds-idrepo-2" 19:51:53 DEBUG }, 19:51:53 DEBUG "value": [ 19:51:53 DEBUG 1713639113, 19:51:53 DEBUG "0" 19:51:53 DEBUG ] 19:51:53 DEBUG } 19:51:53 DEBUG ] 19:51:53 DEBUG } 19:51:53 DEBUG } 19:51:53 INFO [http_cmd]: http status code OK 19:51:53 DEBUG --- status code --- 19:51:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:53 DEBUG --- http response --- 19:51:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:53 DEBUG ----- output ----- 19:51:53 DEBUG { 19:51:53 DEBUG "status": "success", 19:51:53 DEBUG "data": { 19:51:53 DEBUG "resultType": "vector", 19:51:53 DEBUG "result": [ 19:51:53 DEBUG { 19:51:53 DEBUG "metric": { 19:51:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:53 DEBUG }, 19:51:53 DEBUG "value": [ 19:51:53 DEBUG 19:51:53 DEBUG 19:51:53 DEBUG 19:51:53 DEBUG [...] 19:51:53 DEBUG 19:51:53 DEBUG 19:51:53 DEBUG 19:51:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:53 DEBUG }, 19:51:53 DEBUG "value": [ 19:51:53 DEBUG 1713639113, 19:51:53 DEBUG "2663.8" 19:51:53 DEBUG ] 19:51:53 DEBUG } 19:51:53 DEBUG ] 19:51:53 DEBUG } 19:51:53 DEBUG } 19:51:53 INFO 19:51:53 INFO [loop_until]: kubectl --namespace=xlou top pods 19:51:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:51:53 INFO [loop_until]: OK (rc = 0) 19:51:53 DEBUG --- stdout --- 19:51:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17411m 8651Mi am-767f6b67b5-qcqz2 17279m 9064Mi am-767f6b67b5-vbltz 18277m 8665Mi ds-cts-0 19243m 10257Mi ds-cts-1 17924m 10229Mi ds-cts-2 18776m 10304Mi ds-idrepo-0 7530m 3888Mi ds-idrepo-1 7456m 3882Mi ds-idrepo-2 7282m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 17m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8906m 755Mi 19:51:53 DEBUG --- stderr --- 19:51:53 DEBUG 19:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639114" 19:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639114" 19:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639114" 19:51:54 INFO [http_cmd]: http status code OK 19:51:54 DEBUG --- status code --- 19:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:54 DEBUG --- http response --- 19:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:51:54 DEBUG ----- output ----- 19:51:54 DEBUG { 19:51:54 DEBUG "status": "success", 19:51:54 DEBUG "data": { 19:51:54 DEBUG "resultType": "vector", 19:51:54 DEBUG "result": [ 19:51:54 DEBUG { 19:51:54 DEBUG "metric": { 19:51:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:51:54 INFO [http_cmd]: http status code OK 19:51:54 DEBUG --- status code --- 19:51:54 INFO [http_cmd]: http status code OK 19:51:54 DEBUG }, 19:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:54 DEBUG --- status code --- 19:51:54 DEBUG "value": [ 19:51:54 DEBUG --- http response --- 19:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:54 DEBUG 19:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:54 DEBUG --- http response --- 19:51:54 DEBUG 19:51:54 DEBUG ----- output ----- 19:51:54 DEBUG { 19:51:54 DEBUG 19:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:54 DEBUG ----- output ----- 19:51:54 DEBUG { 19:51:54 DEBUG "status": "success", 19:51:54 DEBUG "data": { 19:51:54 DEBUG [...] 19:51:54 DEBUG "resultType": "vector", 19:51:54 DEBUG "status": "success", 19:51:54 DEBUG 19:51:54 DEBUG "result": [ 19:51:54 DEBUG "data": { 19:51:54 DEBUG 19:51:54 DEBUG { 19:51:54 DEBUG "resultType": "vector", 19:51:54 DEBUG 19:51:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:51:54 DEBUG "result": [ 19:51:54 DEBUG "metric": { 19:51:54 DEBUG }, 19:51:54 DEBUG { 19:51:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:51:54 DEBUG "value": [ 19:51:54 DEBUG "metric": { 19:51:54 DEBUG }, 19:51:54 DEBUG 1713639114, 19:51:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:54 DEBUG "value": [ 19:51:54 DEBUG "0" 19:51:54 DEBUG }, 19:51:54 DEBUG 19:51:54 DEBUG ] 19:51:54 DEBUG "value": [ 19:51:54 DEBUG 19:51:54 DEBUG } 19:51:54 DEBUG 19:51:54 DEBUG 19:51:54 DEBUG ] 19:51:54 DEBUG 19:51:54 DEBUG [...] 19:51:54 DEBUG } 19:51:54 DEBUG } 19:51:54 DEBUG 19:51:54 DEBUG 19:51:54 DEBUG 19:51:54 DEBUG [...] 19:51:54 DEBUG 19:51:54 DEBUG 19:51:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:51:54 DEBUG 19:51:54 DEBUG }, 19:51:54 DEBUG 19:51:54 DEBUG "value": [ 19:51:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:54 DEBUG 1713639114, 19:51:54 DEBUG }, 19:51:54 DEBUG "8.911945550664711" 19:51:54 DEBUG "value": [ 19:51:54 DEBUG ] 19:51:54 DEBUG 1713639114, 19:51:54 DEBUG "2658.4" 19:51:54 DEBUG ] 19:51:54 DEBUG } 19:51:54 DEBUG } 19:51:54 DEBUG ] 19:51:54 DEBUG ] 19:51:54 DEBUG } 19:51:54 DEBUG } 19:51:54 DEBUG } 19:51:54 DEBUG } 19:51:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639114" 19:51:54 INFO [http_cmd]: http status code OK 19:51:54 DEBUG --- status code --- 19:51:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:54 DEBUG --- http response --- 19:51:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:54 DEBUG ----- output ----- 19:51:54 DEBUG { 19:51:54 DEBUG "status": "success", 19:51:54 DEBUG "data": { 19:51:54 DEBUG "resultType": "vector", 19:51:54 DEBUG "result": [ 19:51:54 DEBUG { 19:51:54 DEBUG "metric": { 19:51:54 DEBUG "instance": "10.142.0.107:9100" 19:51:54 DEBUG }, 19:51:54 DEBUG "value": [ 19:51:54 DEBUG 19:51:54 DEBUG 19:51:54 DEBUG 19:51:54 DEBUG [...] 19:51:54 DEBUG 19:51:54 DEBUG 19:51:54 DEBUG 19:51:54 DEBUG "instance": "10.142.0.43:9100" 19:51:54 DEBUG }, 19:51:54 DEBUG "value": [ 19:51:54 DEBUG 1713639114, 19:51:54 DEBUG "3615118339072" 19:51:54 DEBUG ] 19:51:54 DEBUG } 19:51:54 DEBUG ] 19:51:54 DEBUG } 19:51:54 DEBUG } 19:51:55 INFO 19:51:55 INFO [loop_until]: kubectl --namespace=xlou top node 19:51:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 DEBUG --- status code --- 19:51:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:55 DEBUG --- http response --- 19:51:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:55 DEBUG ----- output ----- 19:51:55 DEBUG { 19:51:55 DEBUG "status": "success", 19:51:55 DEBUG "data": { 19:51:55 DEBUG "resultType": "vector", 19:51:55 DEBUG "result": [ 19:51:55 DEBUG { 19:51:55 DEBUG "metric": { 19:51:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:55 DEBUG }, 19:51:55 DEBUG "value": [ 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG [...] 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:55 DEBUG }, 19:51:55 DEBUG "value": [ 19:51:55 DEBUG 1713639115, 19:51:55 DEBUG "2658.6" 19:51:55 DEBUG ] 19:51:55 DEBUG } 19:51:55 DEBUG ] 19:51:55 DEBUG } 19:51:55 DEBUG } 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [loop_until]: OK (rc = 0) 19:51:55 DEBUG --- stdout --- 19:51:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6211Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7541m 25% 7633Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16961m 56% 11370Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 92m 0% 3333Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19035m 63% 13485Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5215Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17150m 57% 11952Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14539m 48% 2969Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18814m 63% 10876Mi 9% gke-xlou-cdm-ds-0f601751-j488 8130m 27% 7176Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16589m 55% 15293Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8088m 27% 7855Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19030m 63% 12973Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9114m 57% 6373Mi 10% 19:51:55 DEBUG --- stderr --- 19:51:55 DEBUG 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 DEBUG --- status code --- 19:51:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:55 DEBUG --- http response --- 19:51:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:55 DEBUG ----- output ----- 19:51:55 DEBUG { 19:51:55 DEBUG "status": "success", 19:51:55 DEBUG "data": { 19:51:55 DEBUG "resultType": "vector", 19:51:55 DEBUG "result": [ 19:51:55 DEBUG { 19:51:55 DEBUG "metric": { 19:51:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:55 DEBUG }, 19:51:55 DEBUG "value": [ 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG [...] 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:55 DEBUG }, 19:51:55 DEBUG "value": [ 19:51:55 DEBUG 1713639115, 19:51:55 DEBUG "2658.5333333333333" 19:51:55 DEBUG ] 19:51:55 DEBUG } 19:51:55 DEBUG ] 19:51:55 DEBUG } 19:51:55 DEBUG } 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 DEBUG --- status code --- 19:51:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:55 DEBUG --- http response --- 19:51:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:51:55 DEBUG ----- output ----- 19:51:55 DEBUG { 19:51:55 DEBUG "status": "success", 19:51:55 DEBUG "data": { 19:51:55 DEBUG "resultType": "vector", 19:51:55 DEBUG "result": [ 19:51:55 DEBUG { 19:51:55 DEBUG "metric": { 19:51:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:51:55 DEBUG }, 19:51:55 DEBUG "value": [ 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG [...] 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:51:55 DEBUG }, 19:51:55 DEBUG "value": [ 19:51:55 DEBUG 1713639115, 19:51:55 DEBUG "0.00001648157317678077" 19:51:55 DEBUG ] 19:51:55 DEBUG } 19:51:55 DEBUG ] 19:51:55 DEBUG } 19:51:55 DEBUG } 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 DEBUG --- status code --- 19:51:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:55 DEBUG --- http response --- 19:51:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:55 DEBUG ----- output ----- 19:51:55 DEBUG { 19:51:55 DEBUG "status": "success", 19:51:55 DEBUG "data": { 19:51:55 DEBUG "resultType": "vector", 19:51:55 DEBUG "result": [ 19:51:55 DEBUG { 19:51:55 DEBUG "metric": { 19:51:55 DEBUG "pod": "ds-idrepo-0" 19:51:55 DEBUG }, 19:51:55 DEBUG "value": [ 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG [...] 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG 19:51:55 DEBUG "pod": "ds-idrepo-2" 19:51:55 DEBUG }, 19:51:55 DEBUG "value": [ 19:51:55 DEBUG 1713639115, 19:51:55 DEBUG "0" 19:51:55 DEBUG ] 19:51:55 DEBUG } 19:51:55 DEBUG ] 19:51:55 DEBUG } 19:51:55 DEBUG } 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639115" 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 DEBUG --- status code --- 19:51:55 DEBUG --- status code --- 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 DEBUG --- status code --- 19:51:55 DEBUG --- status code --- 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 INFO [http_cmd]: http status code OK 19:51:55 DEBUG --- status code --- 19:51:56 INFO [http_cmd]: http status code OK 19:51:56 DEBUG --- status code --- 19:51:56 INFO [http_cmd]: http status code OK 19:51:56 DEBUG --- status code --- 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG --- status code --- 19:51:56 DEBUG --- status code --- 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG --- status code --- 19:51:56 DEBUG --- status code --- 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG --- status code --- 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG --- status code --- 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 INFO [http_cmd]: http status code OK 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG --- status code --- 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:51:56 DEBUG { 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG { 19:51:56 DEBUG { 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG { 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG { 19:51:56 DEBUG --- http response --- 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG { 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG { 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG { 19:51:56 DEBUG { 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:51:56 DEBUG { 19:51:56 DEBUG { 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG "data": { 19:51:56 DEBUG "data": { 19:51:56 DEBUG "data": { 19:51:56 DEBUG "data": { 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG "data": { 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG "data": { 19:51:56 DEBUG ----- output ----- 19:51:56 DEBUG { 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG { 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "data": { 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "data": { 19:51:56 DEBUG "data": { 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG { 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG "data": { 19:51:56 DEBUG "data": { 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG "result": [ 19:51:56 DEBUG "result": [ 19:51:56 DEBUG "result": [ 19:51:56 DEBUG "result": [ 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "result": [ 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "result": [ 19:51:56 DEBUG "status": "success", 19:51:56 DEBUG "data": { 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "data": { 19:51:56 DEBUG { 19:51:56 DEBUG { 19:51:56 DEBUG { 19:51:56 DEBUG { 19:51:56 DEBUG "result": [ 19:51:56 DEBUG { 19:51:56 DEBUG "result": [ 19:51:56 DEBUG "result": [ 19:51:56 DEBUG { 19:51:56 DEBUG "data": { 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "result": [ 19:51:56 DEBUG "result": [ 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "metric": { 19:51:56 DEBUG "metric": { 19:51:56 DEBUG "metric": { 19:51:56 DEBUG "metric": { 19:51:56 DEBUG { 19:51:56 DEBUG "metric": { 19:51:56 DEBUG { 19:51:56 DEBUG { 19:51:56 DEBUG "metric": { 19:51:56 DEBUG "resultType": "vector", 19:51:56 DEBUG "result": [ 19:51:56 DEBUG { 19:51:56 DEBUG { 19:51:56 DEBUG "result": [ 19:51:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:51:56 DEBUG }, 19:51:56 DEBUG "instance": "10.142.0.107:9100" 19:51:56 DEBUG "instance": "10.142.0.107:9100" 19:51:56 DEBUG "metric": { 19:51:56 DEBUG "pod": "ds-cts-0" 19:51:56 DEBUG "metric": { 19:51:56 DEBUG "metric": { 19:51:56 DEBUG "pod": "ds-idrepo-0" 19:51:56 DEBUG "result": [ 19:51:56 DEBUG "metric": { 19:51:56 DEBUG { 19:51:56 DEBUG "metric": { 19:51:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:51:56 DEBUG "pod": "ds-cts-0" 19:51:56 DEBUG "value": [ 19:51:56 DEBUG }, 19:51:56 DEBUG }, 19:51:56 DEBUG "instance": "10.142.0.107:9100" 19:51:56 DEBUG }, 19:51:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:56 DEBUG }, 19:51:56 DEBUG "instance": "10.142.0.107:9100" 19:51:56 DEBUG { 19:51:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:51:56 DEBUG "metric": { 19:51:56 DEBUG }, 19:51:56 DEBUG { 19:51:56 DEBUG }, 19:51:56 DEBUG 19:51:56 DEBUG "value": [ 19:51:56 DEBUG "value": [ 19:51:56 DEBUG }, 19:51:56 DEBUG "value": [ 19:51:56 DEBUG }, 19:51:56 DEBUG "value": [ 19:51:56 DEBUG }, 19:51:56 DEBUG "metric": { 19:51:56 DEBUG }, 19:51:56 DEBUG "pod": "ds-cts-0" 19:51:56 DEBUG "value": [ 19:51:56 DEBUG "metric": { 19:51:56 DEBUG "value": [ 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG "value": [ 19:51:56 DEBUG 19:51:56 DEBUG "value": [ 19:51:56 DEBUG 19:51:56 DEBUG "value": [ 19:51:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:56 DEBUG "value": [ 19:51:56 DEBUG }, 19:51:56 DEBUG 19:51:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG }, 19:51:56 DEBUG 19:51:56 DEBUG "value": [ 19:51:56 DEBUG 19:51:56 DEBUG }, 19:51:56 DEBUG "value": [ 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG "value": [ 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG [...] 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG "pod": "ds-cts-2" 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG [...] 19:51:56 DEBUG 19:51:56 DEBUG }, 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG "instance": "10.142.0.43:9100" 19:51:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:51:56 DEBUG 19:51:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:51:56 DEBUG 19:51:56 DEBUG 19:51:56 DEBUG "value": [ 19:51:56 DEBUG 19:51:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:56 DEBUG }, 19:51:56 DEBUG }, 19:51:56 DEBUG "value": [ 19:51:56 DEBUG }, 19:51:56 DEBUG "pod": "ds-cts-2" 19:51:56 DEBUG "pod": "ds-cts-2" 19:51:56 DEBUG }, 19:51:56 DEBUG "value": [ 19:51:56 DEBUG "instance": "10.142.0.43:9100" 19:51:56 DEBUG 19:51:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG "instance": "10.142.0.43:9100" 19:51:56 DEBUG }, 19:51:56 DEBUG "value": [ 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG "instance": "10.142.0.43:9100" 19:51:56 DEBUG "value": [ 19:51:56 DEBUG }, 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG "0" 19:51:56 DEBUG }, 19:51:56 DEBUG }, 19:51:56 DEBUG "value": [ 19:51:56 DEBUG 19:51:56 DEBUG "0.00046390666666666663" 19:51:56 DEBUG }, 19:51:56 DEBUG "value": [ 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG "0" 19:51:56 DEBUG }, 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG "value": [ 19:51:56 DEBUG ] 19:51:56 DEBUG 19:51:56 DEBUG "value": [ 19:51:56 DEBUG }, 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG "pod": "ds-cts-2" 19:51:56 DEBUG ] 19:51:56 DEBUG "value": [ 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG "0.1651666666667448" 19:51:56 DEBUG ] 19:51:56 DEBUG "value": [ 19:51:56 DEBUG "480141" 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG "0" 19:51:56 DEBUG 19:51:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:51:56 DEBUG "value": [ 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG }, 19:51:56 DEBUG "value": [ 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG ] 19:51:56 DEBUG } 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG ] 19:51:56 DEBUG ] 19:51:56 DEBUG } 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG }, 19:51:56 DEBUG "890179584" 19:51:56 DEBUG "80661343.13333333" 19:51:56 DEBUG "80657197.1" 19:51:56 DEBUG "29.966666666666665" 19:51:56 DEBUG } 19:51:56 DEBUG "0" 19:51:56 DEBUG } 19:51:56 DEBUG ] 19:51:56 DEBUG "0.06533333333333076" 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG ] 19:51:56 DEBUG "0" 19:51:56 DEBUG "value": [ 19:51:56 DEBUG ] 19:51:56 DEBUG ] 19:51:56 DEBUG ] 19:51:56 DEBUG ] 19:51:56 DEBUG } 19:51:56 DEBUG ] 19:51:56 DEBUG ] 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG ] 19:51:56 DEBUG ] 19:51:56 DEBUG } 19:51:56 DEBUG ] 19:51:56 DEBUG 1713639115, 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG ] 19:51:56 DEBUG ] 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG ] 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG "0.156" 19:51:56 DEBUG ] 19:51:56 DEBUG } 19:51:56 DEBUG ] 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG ] 19:51:56 DEBUG ] 19:51:56 DEBUG ] 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG ] 19:51:56 DEBUG ] 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG ] 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:51:56 DEBUG } 19:52:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639172" 19:52:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639172" 19:52:52 INFO [http_cmd]: http status code OK 19:52:52 DEBUG --- status code --- 19:52:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:52 DEBUG --- http response --- 19:52:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:52:52 ERROR [http_cmd]: ERROR 19:52:52 ERROR --- status code --- 19:52:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:52:52 ERROR --- http response --- 19:52:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:52:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:52:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639173" 19:52:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639173" 19:52:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639173" 19:52:53 INFO [http_cmd]: http status code OK 19:52:53 DEBUG --- status code --- 19:52:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:53 DEBUG --- http response --- 19:52:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:53 DEBUG ----- output ----- 19:52:53 DEBUG { 19:52:53 DEBUG "status": "success", 19:52:53 DEBUG "data": { 19:52:53 DEBUG "resultType": "vector", 19:52:53 DEBUG "result": [ 19:52:53 DEBUG { 19:52:53 DEBUG "metric": { 19:52:53 DEBUG "pod": "ds-idrepo-0" 19:52:53 DEBUG }, 19:52:53 DEBUG "value": [ 19:52:53 DEBUG 19:52:53 DEBUG 19:52:53 DEBUG 19:52:53 DEBUG [...] 19:52:53 DEBUG 19:52:53 DEBUG 19:52:53 DEBUG 19:52:53 DEBUG "pod": "ds-idrepo-2" 19:52:53 DEBUG }, 19:52:53 DEBUG "value": [ 19:52:53 DEBUG 1713639173, 19:52:53 DEBUG "0" 19:52:53 DEBUG ] 19:52:53 DEBUG } 19:52:53 DEBUG ] 19:52:53 DEBUG } 19:52:53 DEBUG } 19:52:53 INFO [http_cmd]: http status code OK 19:52:53 DEBUG --- status code --- 19:52:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:53 DEBUG --- http response --- 19:52:53 INFO [http_cmd]: http status code OK 19:52:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:53 DEBUG --- status code --- 19:52:53 DEBUG ----- output ----- 19:52:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:53 DEBUG { 19:52:53 DEBUG --- http response --- 19:52:53 DEBUG "status": "success", 19:52:53 DEBUG "data": { 19:52:53 DEBUG "resultType": "vector", 19:52:53 DEBUG "result": [ 19:52:53 DEBUG { 19:52:53 DEBUG "metric": { 19:52:53 DEBUG "pod": "ds-idrepo-0" 19:52:53 DEBUG }, 19:52:53 DEBUG "value": [ 19:52:53 DEBUG 19:52:53 DEBUG 19:52:53 DEBUG 19:52:53 DEBUG [...] 19:52:53 DEBUG 19:52:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:53 DEBUG 19:52:53 DEBUG ----- output ----- 19:52:53 DEBUG 19:52:53 DEBUG { 19:52:53 DEBUG "pod": "ds-idrepo-2" 19:52:53 DEBUG "status": "success", 19:52:53 DEBUG }, 19:52:53 DEBUG "data": { 19:52:53 DEBUG "value": [ 19:52:53 DEBUG "resultType": "vector", 19:52:53 DEBUG 1713639173, 19:52:53 DEBUG "result": [ 19:52:53 DEBUG "0" 19:52:53 DEBUG { 19:52:53 DEBUG ] 19:52:53 DEBUG "metric": { 19:52:53 DEBUG } 19:52:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:53 DEBUG ] 19:52:53 DEBUG }, 19:52:53 DEBUG } 19:52:53 DEBUG "value": [ 19:52:53 DEBUG } 19:52:53 DEBUG 19:52:53 DEBUG 19:52:53 DEBUG 19:52:53 DEBUG [...] 19:52:53 DEBUG 19:52:53 DEBUG 19:52:53 DEBUG 19:52:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:53 DEBUG }, 19:52:53 DEBUG "value": [ 19:52:53 DEBUG 1713639173, 19:52:53 DEBUG "2662.5666666666666" 19:52:53 DEBUG ] 19:52:53 DEBUG } 19:52:53 DEBUG ] 19:52:53 DEBUG } 19:52:53 DEBUG } 19:52:54 INFO 19:52:54 INFO [loop_until]: kubectl --namespace=xlou top pods 19:52:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:52:54 INFO [loop_until]: OK (rc = 0) 19:52:54 DEBUG --- stdout --- 19:52:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16701m 8649Mi am-767f6b67b5-qcqz2 16850m 9063Mi am-767f6b67b5-vbltz 18422m 8668Mi ds-cts-0 18438m 10273Mi ds-cts-1 18318m 10233Mi ds-cts-2 19192m 10300Mi ds-idrepo-0 7339m 3887Mi ds-idrepo-1 7445m 3882Mi ds-idrepo-2 7261m 3779Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8731m 756Mi 19:52:54 DEBUG --- stderr --- 19:52:54 DEBUG 19:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639174" 19:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639174" 19:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639174" 19:52:54 INFO [http_cmd]: http status code OK 19:52:54 DEBUG --- status code --- 19:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:54 DEBUG --- http response --- 19:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:54 DEBUG ----- output ----- 19:52:54 DEBUG { 19:52:54 DEBUG "status": "success", 19:52:54 DEBUG "data": { 19:52:54 DEBUG "resultType": "vector", 19:52:54 DEBUG "result": [ 19:52:54 DEBUG { 19:52:54 DEBUG "metric": { 19:52:54 INFO [http_cmd]: http status code OK 19:52:54 INFO [http_cmd]: http status code OK 19:52:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:54 DEBUG --- status code --- 19:52:54 DEBUG --- status code --- 19:52:54 DEBUG }, 19:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:54 DEBUG "value": [ 19:52:54 DEBUG --- http response --- 19:52:54 DEBUG --- http response --- 19:52:54 DEBUG 19:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG ----- output ----- 19:52:54 DEBUG [...] 19:52:54 DEBUG ----- output ----- 19:52:54 DEBUG { 19:52:54 DEBUG 19:52:54 DEBUG { 19:52:54 DEBUG "status": "success", 19:52:54 DEBUG 19:52:54 DEBUG "status": "success", 19:52:54 DEBUG "data": { 19:52:54 DEBUG 19:52:54 DEBUG "data": { 19:52:54 DEBUG "resultType": "vector", 19:52:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:54 DEBUG "resultType": "vector", 19:52:54 DEBUG "result": [ 19:52:54 DEBUG { 19:52:54 DEBUG "result": [ 19:52:54 DEBUG { 19:52:54 DEBUG "metric": { 19:52:54 DEBUG "metric": { 19:52:54 DEBUG }, 19:52:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:52:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:52:54 DEBUG "value": [ 19:52:54 DEBUG }, 19:52:54 DEBUG }, 19:52:54 DEBUG "value": [ 19:52:54 DEBUG "value": [ 19:52:54 DEBUG 1713639174, 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG "2671.8333333333335" 19:52:54 DEBUG ] 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG } 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG ] 19:52:54 DEBUG } 19:52:54 DEBUG [...] 19:52:54 DEBUG } 19:52:54 DEBUG [...] 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:52:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:52:54 DEBUG }, 19:52:54 DEBUG }, 19:52:54 DEBUG "value": [ 19:52:54 DEBUG "value": [ 19:52:54 DEBUG 1713639174, 19:52:54 DEBUG 1713639174, 19:52:54 DEBUG "8.881820605147347" 19:52:54 DEBUG ] 19:52:54 DEBUG } 19:52:54 DEBUG ] 19:52:54 DEBUG } 19:52:54 DEBUG } 19:52:54 DEBUG "0" 19:52:54 DEBUG ] 19:52:54 DEBUG } 19:52:54 DEBUG ] 19:52:54 DEBUG } 19:52:54 DEBUG } 19:52:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639174" 19:52:54 INFO [http_cmd]: http status code OK 19:52:54 DEBUG --- status code --- 19:52:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:54 DEBUG --- http response --- 19:52:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:54 DEBUG ----- output ----- 19:52:54 DEBUG { 19:52:54 DEBUG "status": "success", 19:52:54 DEBUG "data": { 19:52:54 DEBUG "resultType": "vector", 19:52:54 DEBUG "result": [ 19:52:54 DEBUG { 19:52:54 DEBUG "metric": { 19:52:54 DEBUG "instance": "10.142.0.107:9100" 19:52:54 DEBUG }, 19:52:54 DEBUG "value": [ 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG [...] 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG 19:52:54 DEBUG "instance": "10.142.0.43:9100" 19:52:54 DEBUG }, 19:52:54 DEBUG "value": [ 19:52:54 DEBUG 1713639174, 19:52:54 DEBUG "3619073692672" 19:52:54 DEBUG ] 19:52:54 DEBUG } 19:52:54 DEBUG ] 19:52:54 DEBUG } 19:52:54 DEBUG } 19:52:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639175" 19:52:55 INFO [http_cmd]: http status code OK 19:52:55 DEBUG --- status code --- 19:52:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:55 DEBUG --- http response --- 19:52:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:55 DEBUG ----- output ----- 19:52:55 DEBUG { 19:52:55 DEBUG "status": "success", 19:52:55 DEBUG "data": { 19:52:55 DEBUG "resultType": "vector", 19:52:55 DEBUG "result": [ 19:52:55 DEBUG { 19:52:55 DEBUG "metric": { 19:52:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:55 DEBUG }, 19:52:55 DEBUG "value": [ 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG [...] 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:55 DEBUG }, 19:52:55 DEBUG "value": [ 19:52:55 DEBUG 1713639175, 19:52:55 DEBUG "2670.6" 19:52:55 DEBUG ] 19:52:55 DEBUG } 19:52:55 DEBUG ] 19:52:55 DEBUG } 19:52:55 DEBUG } 19:52:55 INFO 19:52:55 INFO [loop_until]: kubectl --namespace=xlou top node 19:52:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:52:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639175" 19:52:55 INFO [http_cmd]: http status code OK 19:52:55 DEBUG --- status code --- 19:52:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:55 DEBUG --- http response --- 19:52:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:55 DEBUG ----- output ----- 19:52:55 DEBUG { 19:52:55 DEBUG "status": "success", 19:52:55 DEBUG "data": { 19:52:55 DEBUG "resultType": "vector", 19:52:55 DEBUG "result": [ 19:52:55 DEBUG { 19:52:55 DEBUG "metric": { 19:52:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:55 DEBUG }, 19:52:55 DEBUG "value": [ 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG [...] 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:55 DEBUG }, 19:52:55 DEBUG "value": [ 19:52:55 DEBUG 1713639175, 19:52:55 DEBUG "2672.1666666666665" 19:52:55 DEBUG ] 19:52:55 DEBUG } 19:52:55 DEBUG ] 19:52:55 DEBUG } 19:52:55 DEBUG } 19:52:55 INFO [loop_until]: OK (rc = 0) 19:52:55 DEBUG --- stdout --- 19:52:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6209Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7559m 25% 7630Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16880m 56% 11362Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3345Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18584m 62% 13497Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5231Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17186m 57% 11958Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14539m 48% 2962Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19496m 65% 10882Mi 9% gke-xlou-cdm-ds-0f601751-j488 8167m 27% 7169Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17210m 57% 15296Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8126m 27% 7845Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18910m 63% 12964Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9088m 57% 6373Mi 10% 19:52:55 DEBUG --- stderr --- 19:52:55 DEBUG 19:52:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639175" 19:52:55 INFO [http_cmd]: http status code OK 19:52:55 DEBUG --- status code --- 19:52:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:55 DEBUG --- http response --- 19:52:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:52:55 DEBUG ----- output ----- 19:52:55 DEBUG { 19:52:55 DEBUG "status": "success", 19:52:55 DEBUG "data": { 19:52:55 DEBUG "resultType": "vector", 19:52:55 DEBUG "result": [ 19:52:55 DEBUG { 19:52:55 DEBUG "metric": { 19:52:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:52:55 DEBUG }, 19:52:55 DEBUG "value": [ 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG [...] 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:52:55 DEBUG }, 19:52:55 DEBUG "value": [ 19:52:55 DEBUG 1713639175, 19:52:55 DEBUG "5.285188278782075" 19:52:55 DEBUG ] 19:52:55 DEBUG } 19:52:55 DEBUG ] 19:52:55 DEBUG } 19:52:55 DEBUG } 19:52:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639175" 19:52:55 INFO [http_cmd]: http status code OK 19:52:55 DEBUG --- status code --- 19:52:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:55 DEBUG --- http response --- 19:52:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:55 DEBUG ----- output ----- 19:52:55 DEBUG { 19:52:55 DEBUG "status": "success", 19:52:55 DEBUG "data": { 19:52:55 DEBUG "resultType": "vector", 19:52:55 DEBUG "result": [ 19:52:55 DEBUG { 19:52:55 DEBUG "metric": { 19:52:55 DEBUG "pod": "ds-idrepo-0" 19:52:55 DEBUG }, 19:52:55 DEBUG "value": [ 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG [...] 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG 19:52:55 DEBUG "pod": "ds-idrepo-2" 19:52:55 DEBUG }, 19:52:55 DEBUG "value": [ 19:52:55 DEBUG 1713639175, 19:52:55 DEBUG "0" 19:52:55 DEBUG ] 19:52:55 DEBUG } 19:52:55 DEBUG ] 19:52:55 DEBUG } 19:52:55 DEBUG } 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639176" 19:52:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639176" 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 DEBUG --- status code --- 19:52:56 DEBUG --- status code --- 19:52:56 DEBUG --- status code --- 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 DEBUG --- status code --- 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 DEBUG --- status code --- 19:52:56 DEBUG --- status code --- 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG --- status code --- 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG --- status code --- 19:52:56 DEBUG --- status code --- 19:52:56 DEBUG --- status code --- 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG --- status code --- 19:52:56 INFO [http_cmd]: http status code OK 19:52:56 DEBUG --- status code --- 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG --- status code --- 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG --- status code --- 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG { 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:56 DEBUG --- http response --- 19:52:56 DEBUG "data": { 19:52:56 DEBUG { 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:52:56 DEBUG { 19:52:56 DEBUG { 19:52:56 DEBUG { 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG { 19:52:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG { 19:52:56 DEBUG { 19:52:56 DEBUG { 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG { 19:52:56 DEBUG { 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG ----- output ----- 19:52:56 DEBUG "result": [ 19:52:56 DEBUG "data": { 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG { 19:52:56 DEBUG { 19:52:56 DEBUG "data": { 19:52:56 DEBUG "data": { 19:52:56 DEBUG "data": { 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG "data": { 19:52:56 DEBUG { 19:52:56 DEBUG { 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "data": { 19:52:56 DEBUG "data": { 19:52:56 DEBUG "data": { 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "data": { 19:52:56 DEBUG "data": { 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "status": "success", 19:52:56 DEBUG "metric": { 19:52:56 DEBUG "result": [ 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "data": { 19:52:56 DEBUG "data": { 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "result": [ 19:52:56 DEBUG "result": [ 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "result": [ 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "data": { 19:52:56 DEBUG "instance": "10.142.0.107:9100" 19:52:56 DEBUG { 19:52:56 DEBUG "result": [ 19:52:56 DEBUG "result": [ 19:52:56 DEBUG "result": [ 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG "result": [ 19:52:56 DEBUG "result": [ 19:52:56 DEBUG { 19:52:56 DEBUG { 19:52:56 DEBUG "metric": { 19:52:56 DEBUG { 19:52:56 DEBUG "result": [ 19:52:56 DEBUG "resultType": "vector", 19:52:56 DEBUG }, 19:52:56 DEBUG "metric": { 19:52:56 DEBUG { 19:52:56 DEBUG { 19:52:56 DEBUG { 19:52:56 DEBUG "result": [ 19:52:56 DEBUG { 19:52:56 DEBUG { 19:52:56 DEBUG "metric": { 19:52:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:52:56 DEBUG "result": [ 19:52:56 DEBUG "metric": { 19:52:56 DEBUG { 19:52:56 DEBUG "result": [ 19:52:56 DEBUG "value": [ 19:52:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:56 DEBUG "metric": { 19:52:56 DEBUG "metric": { 19:52:56 DEBUG "metric": { 19:52:56 DEBUG { 19:52:56 DEBUG "metric": { 19:52:56 DEBUG "metric": { 19:52:56 DEBUG "instance": "10.142.0.107:9100" 19:52:56 DEBUG }, 19:52:56 DEBUG "value": [ 19:52:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:56 DEBUG "metric": { 19:52:56 DEBUG { 19:52:56 DEBUG 19:52:56 DEBUG }, 19:52:56 DEBUG "value": [ 19:52:56 DEBUG "pod": "ds-cts-0" 19:52:56 DEBUG "pod": "ds-idrepo-1" 19:52:56 DEBUG "metric": { 19:52:56 DEBUG "instance": "10.142.0.107:9100" 19:52:56 DEBUG "pod": "ds-cts-0" 19:52:56 DEBUG }, 19:52:56 DEBUG 19:52:56 DEBUG }, 19:52:56 DEBUG "instance": "10.142.0.107:9100" 19:52:56 DEBUG "metric": { 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG "pod": "ds-cts-0" 19:52:56 DEBUG }, 19:52:56 DEBUG }, 19:52:56 DEBUG "pod": "ds-idrepo-0" 19:52:56 DEBUG }, 19:52:56 DEBUG }, 19:52:56 DEBUG { 19:52:56 DEBUG "value": [ 19:52:56 DEBUG 19:52:56 DEBUG "value": [ 19:52:56 DEBUG }, 19:52:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG }, 19:52:56 DEBUG "value": [ 19:52:56 DEBUG "value": [ 19:52:56 DEBUG }, 19:52:56 DEBUG "value": [ 19:52:56 DEBUG "value": [ 19:52:56 DEBUG "metric": { 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG "value": [ 19:52:56 DEBUG }, 19:52:56 DEBUG [...] 19:52:56 DEBUG 19:52:56 DEBUG "value": [ 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG "value": [ 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG "pod": "ds-idrepo-1" 19:52:56 DEBUG 19:52:56 DEBUG [...] 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG "value": [ 19:52:56 DEBUG 19:52:56 DEBUG [...] 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG }, 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG "value": [ 19:52:56 DEBUG [...] 19:52:56 DEBUG 19:52:56 DEBUG [...] 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG [...] 19:52:56 DEBUG [...] 19:52:56 DEBUG 19:52:56 DEBUG [...] 19:52:56 DEBUG [...] 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG [...] 19:52:56 DEBUG 19:52:56 DEBUG "instance": "10.142.0.43:9100" 19:52:56 DEBUG 19:52:56 DEBUG [...] 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG [...] 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:52:56 DEBUG }, 19:52:56 DEBUG 19:52:56 DEBUG [...] 19:52:56 DEBUG }, 19:52:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG [...] 19:52:56 DEBUG 19:52:56 DEBUG "value": [ 19:52:56 DEBUG 19:52:56 DEBUG "instance": "10.142.0.43:9100" 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG }, 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG "pod": "ds-cts-2" 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG 19:52:56 DEBUG }, 19:52:56 DEBUG "value": [ 19:52:56 DEBUG 19:52:56 DEBUG "value": [ 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG }, 19:52:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:52:56 DEBUG }, 19:52:56 DEBUG "pod": "ds-cts-2" 19:52:56 DEBUG 19:52:56 DEBUG 19:52:56 DEBUG "891826176" 19:52:56 DEBUG 19:52:56 DEBUG "value": [ 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG "pod": "ds-cts-2" 19:52:56 DEBUG "pod": "ds-cts-2" 19:52:56 DEBUG "value": [ 19:52:56 DEBUG "value": [ 19:52:56 DEBUG 19:52:56 DEBUG }, 19:52:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:52:56 DEBUG 19:52:56 DEBUG ] 19:52:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG "0.161466666666603" 19:52:56 DEBUG }, 19:52:56 DEBUG "479689.5" 19:52:56 DEBUG }, 19:52:56 DEBUG }, 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG "instance": "10.142.0.43:9100" 19:52:56 DEBUG "value": [ 19:52:56 DEBUG }, 19:52:56 DEBUG "instance": "10.142.0.43:9100" 19:52:56 DEBUG } 19:52:56 DEBUG }, 19:52:56 DEBUG "80876225.3" 19:52:56 DEBUG ] 19:52:56 DEBUG "value": [ 19:52:56 DEBUG ] 19:52:56 DEBUG "value": [ 19:52:56 DEBUG "value": [ 19:52:56 DEBUG "0" 19:52:56 DEBUG "0" 19:52:56 DEBUG }, 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG "value": [ 19:52:56 DEBUG }, 19:52:56 DEBUG ] 19:52:56 DEBUG "value": [ 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG "0" 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG ] 19:52:56 DEBUG ] 19:52:56 DEBUG "value": [ 19:52:56 DEBUG "30" 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG "value": [ 19:52:56 DEBUG } 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG "0.00050384" 19:52:56 DEBUG "0" 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG ] 19:52:56 DEBUG "0" 19:52:56 DEBUG 1713639176, 19:52:56 DEBUG } 19:52:56 DEBUG "0.156" 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG ] 19:52:56 DEBUG ] 19:52:56 DEBUG ] 19:52:56 DEBUG "80880553.30000001" 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG "0.06333333333334584" 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG ] 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:52:56 DEBUG } 19:53:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639232" 19:53:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639232" 19:53:52 ERROR [http_cmd]: ERROR 19:53:52 INFO [http_cmd]: http status code OK 19:53:52 ERROR --- status code --- 19:53:52 DEBUG --- status code --- 19:53:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:53:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:52 ERROR --- http response --- 19:53:52 DEBUG --- http response --- 19:53:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:53:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:53:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:53:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639233" 19:53:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639233" 19:53:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639233" 19:53:53 INFO [http_cmd]: http status code OK 19:53:53 DEBUG --- status code --- 19:53:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:53 DEBUG --- http response --- 19:53:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:53 DEBUG ----- output ----- 19:53:53 DEBUG { 19:53:53 DEBUG "status": "success", 19:53:53 DEBUG "data": { 19:53:53 DEBUG "resultType": "vector", 19:53:53 DEBUG "result": [ 19:53:53 DEBUG { 19:53:53 DEBUG "metric": { 19:53:53 DEBUG "pod": "ds-idrepo-0" 19:53:53 DEBUG }, 19:53:53 DEBUG "value": [ 19:53:53 DEBUG 19:53:53 DEBUG 19:53:53 DEBUG 19:53:53 DEBUG [...] 19:53:53 DEBUG 19:53:53 DEBUG 19:53:53 DEBUG 19:53:53 DEBUG "pod": "ds-idrepo-2" 19:53:53 DEBUG }, 19:53:53 DEBUG "value": [ 19:53:53 DEBUG 1713639233, 19:53:53 DEBUG "0" 19:53:53 DEBUG ] 19:53:53 DEBUG } 19:53:53 DEBUG ] 19:53:53 DEBUG } 19:53:53 DEBUG } 19:53:53 INFO [http_cmd]: http status code OK 19:53:53 DEBUG --- status code --- 19:53:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:53 DEBUG --- http response --- 19:53:53 INFO [http_cmd]: http status code OK 19:53:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:53 DEBUG ----- output ----- 19:53:53 DEBUG { 19:53:53 DEBUG "status": "success", 19:53:53 DEBUG "data": { 19:53:53 DEBUG --- status code --- 19:53:53 DEBUG "resultType": "vector", 19:53:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:53 DEBUG "result": [ 19:53:53 DEBUG --- http response --- 19:53:53 DEBUG { 19:53:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:53 DEBUG "metric": { 19:53:53 DEBUG ----- output ----- 19:53:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:53 DEBUG { 19:53:53 DEBUG }, 19:53:53 DEBUG "status": "success", 19:53:53 DEBUG "value": [ 19:53:53 DEBUG 19:53:53 DEBUG 19:53:53 DEBUG 19:53:53 DEBUG [...] 19:53:53 DEBUG 19:53:53 DEBUG 19:53:53 DEBUG 19:53:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:53 DEBUG }, 19:53:53 DEBUG "value": [ 19:53:53 DEBUG 1713639233, 19:53:53 DEBUG "2670.2" 19:53:53 DEBUG ] 19:53:53 DEBUG } 19:53:53 DEBUG ] 19:53:53 DEBUG } 19:53:53 DEBUG } 19:53:53 DEBUG "data": { 19:53:53 DEBUG "resultType": "vector", 19:53:53 DEBUG "result": [ 19:53:53 DEBUG { 19:53:53 DEBUG "metric": { 19:53:53 DEBUG "pod": "ds-idrepo-0" 19:53:53 DEBUG }, 19:53:53 DEBUG "value": [ 19:53:53 DEBUG 19:53:53 DEBUG 19:53:53 DEBUG 19:53:53 DEBUG [...] 19:53:53 DEBUG 19:53:53 DEBUG 19:53:53 DEBUG 19:53:53 DEBUG "pod": "ds-idrepo-2" 19:53:53 DEBUG }, 19:53:53 DEBUG "value": [ 19:53:53 DEBUG 1713639233, 19:53:53 DEBUG "0" 19:53:53 DEBUG ] 19:53:53 DEBUG } 19:53:53 DEBUG ] 19:53:53 DEBUG } 19:53:53 DEBUG } 19:53:54 INFO 19:53:54 INFO [loop_until]: kubectl --namespace=xlou top pods 19:53:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:53:54 INFO [loop_until]: OK (rc = 0) 19:53:54 DEBUG --- stdout --- 19:53:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17127m 8649Mi am-767f6b67b5-qcqz2 16858m 9064Mi am-767f6b67b5-vbltz 18185m 8667Mi ds-cts-0 18920m 10286Mi ds-cts-1 16750m 10268Mi ds-cts-2 18989m 10345Mi ds-idrepo-0 7277m 3888Mi ds-idrepo-1 7172m 3882Mi ds-idrepo-2 7373m 3778Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 19m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9023m 759Mi 19:53:54 DEBUG --- stderr --- 19:53:54 DEBUG 19:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639234" 19:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639234" 19:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639234" 19:53:54 INFO [http_cmd]: http status code OK 19:53:54 INFO [http_cmd]: http status code OK 19:53:54 DEBUG --- status code --- 19:53:54 INFO [http_cmd]: http status code OK 19:53:54 DEBUG --- status code --- 19:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:54 DEBUG --- status code --- 19:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:54 DEBUG --- http response --- 19:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:54 DEBUG --- http response --- 19:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:54 DEBUG --- http response --- 19:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:53:54 DEBUG ----- output ----- 19:53:54 DEBUG ----- output ----- 19:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:54 DEBUG { 19:53:54 DEBUG { 19:53:54 DEBUG "status": "success", 19:53:54 DEBUG "status": "success", 19:53:54 DEBUG "data": { 19:53:54 DEBUG ----- output ----- 19:53:54 DEBUG "data": { 19:53:54 DEBUG "resultType": "vector", 19:53:54 DEBUG { 19:53:54 DEBUG "resultType": "vector", 19:53:54 DEBUG "result": [ 19:53:54 DEBUG "status": "success", 19:53:54 DEBUG "result": [ 19:53:54 DEBUG { 19:53:54 DEBUG "data": { 19:53:54 DEBUG { 19:53:54 DEBUG "metric": { 19:53:54 DEBUG "resultType": "vector", 19:53:54 DEBUG "metric": { 19:53:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:53:54 DEBUG "result": [ 19:53:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:54 DEBUG }, 19:53:54 DEBUG { 19:53:54 DEBUG }, 19:53:54 DEBUG "value": [ 19:53:54 DEBUG "metric": { 19:53:54 DEBUG "value": [ 19:53:54 DEBUG 19:53:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:53:54 DEBUG 19:53:54 DEBUG 19:53:54 DEBUG }, 19:53:54 DEBUG 19:53:54 DEBUG 19:53:54 DEBUG "value": [ 19:53:54 DEBUG 19:53:54 DEBUG [...] 19:53:54 DEBUG 19:53:54 DEBUG [...] 19:53:54 DEBUG 19:53:54 DEBUG 19:53:54 DEBUG 19:53:54 DEBUG 19:53:54 DEBUG 19:53:54 DEBUG 19:53:54 DEBUG 19:53:54 DEBUG [...] 19:53:54 DEBUG 19:53:54 DEBUG "pod": "idm-b85f9d978-txpwv" 19:53:54 DEBUG 19:53:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:54 DEBUG }, 19:53:54 DEBUG 19:53:54 DEBUG }, 19:53:54 DEBUG "value": [ 19:53:54 DEBUG 19:53:54 DEBUG "value": [ 19:53:54 DEBUG 1713639234, 19:53:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:53:54 DEBUG }, 19:53:54 DEBUG "357.35473739312505" 19:53:54 DEBUG "value": [ 19:53:54 DEBUG 1713639234, 19:53:54 DEBUG "2671.0333333333333" 19:53:54 DEBUG 1713639234, 19:53:54 DEBUG ] 19:53:54 DEBUG ] 19:53:54 DEBUG "8.906331741693114" 19:53:54 DEBUG } 19:53:54 DEBUG } 19:53:54 DEBUG ] 19:53:54 DEBUG ] 19:53:54 DEBUG ] 19:53:54 DEBUG } 19:53:54 DEBUG } 19:53:54 DEBUG } 19:53:54 DEBUG ] 19:53:54 DEBUG } 19:53:54 DEBUG } 19:53:54 DEBUG } 19:53:54 DEBUG } 19:53:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639234" 19:53:54 INFO [http_cmd]: http status code OK 19:53:54 DEBUG --- status code --- 19:53:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:54 DEBUG --- http response --- 19:53:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:54 DEBUG ----- output ----- 19:53:54 DEBUG { 19:53:54 DEBUG "status": "success", 19:53:54 DEBUG "data": { 19:53:54 DEBUG "resultType": "vector", 19:53:54 DEBUG "result": [ 19:53:54 DEBUG { 19:53:54 DEBUG "metric": { 19:53:54 DEBUG "instance": "10.142.0.107:9100" 19:53:54 DEBUG }, 19:53:54 DEBUG "value": [ 19:53:54 DEBUG 19:53:54 DEBUG 19:53:54 DEBUG 19:53:54 DEBUG [...] 19:53:54 DEBUG 19:53:54 DEBUG 19:53:54 DEBUG 19:53:54 DEBUG "instance": "10.142.0.43:9100" 19:53:54 DEBUG }, 19:53:54 DEBUG "value": [ 19:53:54 DEBUG 1713639234, 19:53:54 DEBUG "3622485734400" 19:53:54 DEBUG ] 19:53:54 DEBUG } 19:53:54 DEBUG ] 19:53:54 DEBUG } 19:53:54 DEBUG } 19:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639235" 19:53:55 INFO [http_cmd]: http status code OK 19:53:55 DEBUG --- status code --- 19:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:55 DEBUG --- http response --- 19:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:55 DEBUG ----- output ----- 19:53:55 DEBUG { 19:53:55 DEBUG "status": "success", 19:53:55 DEBUG "data": { 19:53:55 DEBUG "resultType": "vector", 19:53:55 DEBUG "result": [ 19:53:55 DEBUG { 19:53:55 DEBUG "metric": { 19:53:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:55 DEBUG }, 19:53:55 DEBUG "value": [ 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG [...] 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:55 DEBUG }, 19:53:55 DEBUG "value": [ 19:53:55 DEBUG 1713639235, 19:53:55 DEBUG "2670.6666666666665" 19:53:55 DEBUG ] 19:53:55 DEBUG } 19:53:55 DEBUG ] 19:53:55 DEBUG } 19:53:55 DEBUG } 19:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639235" 19:53:55 INFO [http_cmd]: http status code OK 19:53:55 DEBUG --- status code --- 19:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:55 DEBUG --- http response --- 19:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:55 DEBUG ----- output ----- 19:53:55 DEBUG { 19:53:55 DEBUG "status": "success", 19:53:55 DEBUG "data": { 19:53:55 DEBUG "resultType": "vector", 19:53:55 DEBUG "result": [ 19:53:55 DEBUG { 19:53:55 DEBUG "metric": { 19:53:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:55 DEBUG }, 19:53:55 DEBUG "value": [ 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG [...] 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:55 DEBUG }, 19:53:55 DEBUG "value": [ 19:53:55 DEBUG 1713639235, 19:53:55 DEBUG "2670.6" 19:53:55 DEBUG ] 19:53:55 DEBUG } 19:53:55 DEBUG ] 19:53:55 DEBUG } 19:53:55 DEBUG } 19:53:55 INFO 19:53:55 INFO [loop_until]: kubectl --namespace=xlou top node 19:53:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:53:55 INFO [loop_until]: OK (rc = 0) 19:53:55 DEBUG --- stdout --- 19:53:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6223Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7585m 25% 7625Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16876m 56% 11365Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3342Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19114m 64% 13482Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5230Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17226m 57% 11954Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14541m 48% 2970Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19398m 64% 10885Mi 9% gke-xlou-cdm-ds-0f601751-j488 8131m 27% 7174Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16753m 56% 15318Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8086m 27% 7856Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19016m 63% 12996Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9112m 57% 6370Mi 10% 19:53:55 DEBUG --- stderr --- 19:53:55 DEBUG 19:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639235" 19:53:55 INFO [http_cmd]: http status code OK 19:53:55 DEBUG --- status code --- 19:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:55 DEBUG --- http response --- 19:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:53:55 DEBUG ----- output ----- 19:53:55 DEBUG { 19:53:55 DEBUG "status": "success", 19:53:55 DEBUG "data": { 19:53:55 DEBUG "resultType": "vector", 19:53:55 DEBUG "result": [ 19:53:55 DEBUG { 19:53:55 DEBUG "metric": { 19:53:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:53:55 DEBUG }, 19:53:55 DEBUG "value": [ 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG [...] 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG "pod": "idm-b85f9d978-txpwv" 19:53:55 DEBUG }, 19:53:55 DEBUG "value": [ 19:53:55 DEBUG 1713639235, 19:53:55 DEBUG "0.003242162507997907" 19:53:55 DEBUG ] 19:53:55 DEBUG } 19:53:55 DEBUG ] 19:53:55 DEBUG } 19:53:55 DEBUG } 19:53:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639235" 19:53:55 INFO [http_cmd]: http status code OK 19:53:55 DEBUG --- status code --- 19:53:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:55 DEBUG --- http response --- 19:53:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:55 DEBUG ----- output ----- 19:53:55 DEBUG { 19:53:55 DEBUG "status": "success", 19:53:55 DEBUG "data": { 19:53:55 DEBUG "resultType": "vector", 19:53:55 DEBUG "result": [ 19:53:55 DEBUG { 19:53:55 DEBUG "metric": { 19:53:55 DEBUG "pod": "ds-idrepo-0" 19:53:55 DEBUG }, 19:53:55 DEBUG "value": [ 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG [...] 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG 19:53:55 DEBUG "pod": "ds-idrepo-2" 19:53:55 DEBUG }, 19:53:55 DEBUG "value": [ 19:53:55 DEBUG 1713639235, 19:53:55 DEBUG "0" 19:53:55 DEBUG ] 19:53:55 DEBUG } 19:53:55 DEBUG ] 19:53:55 DEBUG } 19:53:55 DEBUG } 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639236" 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 DEBUG --- status code --- 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG { 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG "data": { 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG "result": [ 19:53:56 DEBUG { 19:53:56 DEBUG "metric": { 19:53:56 DEBUG "pod": "ds-cts-0" 19:53:56 DEBUG }, 19:53:56 DEBUG "value": [ 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG [...] 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG "pod": "ds-cts-2" 19:53:56 DEBUG }, 19:53:56 DEBUG "value": [ 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG "0" 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG } 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639236" 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639236" 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639236" 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639236" 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639236" 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639236" 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639236" 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639236" 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639236" 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639236" 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639236" 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639236" 19:53:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639236" 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 DEBUG --- status code --- 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 DEBUG --- status code --- 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 DEBUG --- status code --- 19:53:56 DEBUG --- status code --- 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 DEBUG --- status code --- 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG --- status code --- 19:53:56 DEBUG --- status code --- 19:53:56 DEBUG --- status code --- 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG --- status code --- 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG --- status code --- 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG --- status code --- 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG --- http response --- 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:56 DEBUG ----- output ----- 19:53:56 INFO [http_cmd]: http status code OK 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG { 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:53:56 DEBUG --- status code --- 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG { 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG { 19:53:56 DEBUG --- status code --- 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG "data": { 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG { 19:53:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:53:56 DEBUG { 19:53:56 DEBUG { 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG { 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG { 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG "data": { 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG "data": { 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG "data": { 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG "result": [ 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG { 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG { 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG --- http response --- 19:53:56 DEBUG "result": [ 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG "data": { 19:53:56 DEBUG { 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG "data": { 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG "data": { 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG "result": [ 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG "result": [ 19:53:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:53:56 DEBUG { 19:53:56 DEBUG "data": { 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG "result": [ 19:53:56 DEBUG "data": { 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG { 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG "result": [ 19:53:56 DEBUG { 19:53:56 DEBUG "data": { 19:53:56 DEBUG { 19:53:56 DEBUG ----- output ----- 19:53:56 DEBUG "metric": { 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG { 19:53:56 DEBUG { 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG "result": [ 19:53:56 DEBUG { 19:53:56 DEBUG "data": { 19:53:56 DEBUG "metric": { 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG { 19:53:56 DEBUG "metric": { 19:53:56 DEBUG { 19:53:56 DEBUG "instance": "10.142.0.107:9100" 19:53:56 DEBUG "result": [ 19:53:56 DEBUG "metric": { 19:53:56 DEBUG "metric": { 19:53:56 DEBUG "result": [ 19:53:56 DEBUG "metric": { 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG "result": [ 19:53:56 DEBUG "data": { 19:53:56 DEBUG "metric": { 19:53:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:53:56 DEBUG "status": "success", 19:53:56 DEBUG }, 19:53:56 DEBUG { 19:53:56 DEBUG "pod": "ds-idrepo-0" 19:53:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:56 DEBUG { 19:53:56 DEBUG "pod": "ds-cts-0" 19:53:56 DEBUG "result": [ 19:53:56 DEBUG "pod": "ds-cts-0" 19:53:56 DEBUG { 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG "instance": "10.142.0.107:9100" 19:53:56 DEBUG }, 19:53:56 DEBUG "data": { 19:53:56 DEBUG "value": [ 19:53:56 DEBUG "metric": { 19:53:56 DEBUG }, 19:53:56 DEBUG "value": [ 19:53:56 DEBUG "metric": { 19:53:56 DEBUG }, 19:53:56 DEBUG { 19:53:56 DEBUG }, 19:53:56 DEBUG "metric": { 19:53:56 DEBUG "result": [ 19:53:56 DEBUG }, 19:53:56 DEBUG "value": [ 19:53:56 DEBUG "resultType": "vector", 19:53:56 DEBUG 19:53:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:53:56 DEBUG 19:53:56 DEBUG }, 19:53:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:53:56 DEBUG "value": [ 19:53:56 DEBUG "metric": { 19:53:56 DEBUG "value": [ 19:53:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:56 DEBUG { 19:53:56 DEBUG 19:53:56 DEBUG "value": [ 19:53:56 DEBUG "result": [ 19:53:56 DEBUG 19:53:56 DEBUG }, 19:53:56 DEBUG 19:53:56 DEBUG "value": [ 19:53:56 DEBUG }, 19:53:56 DEBUG 19:53:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:53:56 DEBUG 19:53:56 DEBUG }, 19:53:56 DEBUG "metric": { 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG { 19:53:56 DEBUG 19:53:56 DEBUG "value": [ 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG "value": [ 19:53:56 DEBUG 19:53:56 DEBUG }, 19:53:56 DEBUG 19:53:56 DEBUG "value": [ 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG "metric": { 19:53:56 DEBUG [...] 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG "value": [ 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG "instance": "10.142.0.107:9100" 19:53:56 DEBUG [...] 19:53:56 DEBUG 19:53:56 DEBUG "instance": "10.142.0.107:9100" 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG [...] 19:53:56 DEBUG [...] 19:53:56 DEBUG [...] 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG }, 19:53:56 DEBUG "value": [ 19:53:56 DEBUG [...] 19:53:56 DEBUG }, 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG [...] 19:53:56 DEBUG [...] 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG "value": [ 19:53:56 DEBUG "instance": "10.142.0.43:9100" 19:53:56 DEBUG [...] 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG [...] 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG }, 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG "pod": "ds-cts-2" 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG "value": [ 19:53:56 DEBUG 19:53:56 DEBUG "pod": "ds-cts-2" 19:53:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:56 DEBUG }, 19:53:56 DEBUG "value": [ 19:53:56 DEBUG [...] 19:53:56 DEBUG 19:53:56 DEBUG "pod": "ds-cts-2" 19:53:56 DEBUG "instance": "10.142.0.43:9100" 19:53:56 DEBUG }, 19:53:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:53:56 DEBUG 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG 19:53:56 DEBUG }, 19:53:56 DEBUG }, 19:53:56 DEBUG 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG 19:53:56 DEBUG 19:53:56 DEBUG }, 19:53:56 DEBUG "value": [ 19:53:56 DEBUG [...] 19:53:56 DEBUG }, 19:53:56 DEBUG [...] 19:53:56 DEBUG "80579315.9" 19:53:56 DEBUG "pod": "idm-b85f9d978-txpwv" 19:53:56 DEBUG "value": [ 19:53:56 DEBUG "value": [ 19:53:56 DEBUG 19:53:56 DEBUG "36.3" 19:53:56 DEBUG 19:53:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:56 DEBUG "value": [ 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG 19:53:56 DEBUG "value": [ 19:53:56 DEBUG 19:53:56 DEBUG ] 19:53:56 DEBUG }, 19:53:56 DEBUG "value": [ 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:53:56 DEBUG ] 19:53:56 DEBUG 19:53:56 DEBUG }, 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG "0.1547333333332536" 19:53:56 DEBUG 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG "893403136" 19:53:56 DEBUG } 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG "0" 19:53:56 DEBUG }, 19:53:56 DEBUG } 19:53:56 DEBUG "pod": "idm-b85f9d978-txpwv" 19:53:56 DEBUG "value": [ 19:53:56 DEBUG "0.0001" 19:53:56 DEBUG ] 19:53:56 DEBUG 19:53:56 DEBUG ] 19:53:56 DEBUG 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG "0" 19:53:56 DEBUG ] 19:53:56 DEBUG "value": [ 19:53:56 DEBUG ] 19:53:56 DEBUG }, 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG } 19:53:56 DEBUG ] 19:53:56 DEBUG "instance": "10.142.0.43:9100" 19:53:56 DEBUG } 19:53:56 DEBUG 19:53:56 DEBUG } 19:53:56 DEBUG "0" 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG } 19:53:56 DEBUG "value": [ 19:53:56 DEBUG "0.156" 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG }, 19:53:56 DEBUG ] 19:53:56 DEBUG "instance": "10.142.0.43:9100" 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG ] 19:53:56 DEBUG "480032.5" 19:53:56 DEBUG } 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG } 19:53:56 DEBUG }, 19:53:56 DEBUG } 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG } 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG } 19:53:56 DEBUG } 19:53:56 DEBUG "value": [ 19:53:56 DEBUG } 19:53:56 DEBUG "value": [ 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG "36081988.71633804" 19:53:56 DEBUG } 19:53:56 DEBUG ] 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG 1713639236, 19:53:56 DEBUG } 19:53:56 DEBUG } 19:53:56 DEBUG ] 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG "0.06066666666666076" 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG } 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG } 19:53:56 DEBUG } 19:53:56 DEBUG } 19:53:56 DEBUG "80583704.53333333" 19:53:56 DEBUG } 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG } 19:53:56 DEBUG } 19:53:56 DEBUG ] 19:53:56 DEBUG } 19:53:56 DEBUG } 19:54:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639292" 19:54:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639292" 19:54:52 ERROR [http_cmd]: ERROR 19:54:52 INFO [http_cmd]: http status code OK 19:54:52 ERROR --- status code --- 19:54:52 DEBUG --- status code --- 19:54:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:54:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:52 ERROR --- http response --- 19:54:52 DEBUG --- http response --- 19:54:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:54:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:54:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:54:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639293" 19:54:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639293" 19:54:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639293" 19:54:53 INFO [http_cmd]: http status code OK 19:54:53 INFO [http_cmd]: http status code OK 19:54:53 INFO [http_cmd]: http status code OK 19:54:53 DEBUG --- status code --- 19:54:53 DEBUG --- status code --- 19:54:53 DEBUG --- status code --- 19:54:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:53 DEBUG --- http response --- 19:54:53 DEBUG --- http response --- 19:54:53 DEBUG --- http response --- 19:54:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:53 DEBUG ----- output ----- 19:54:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:53 DEBUG ----- output ----- 19:54:53 DEBUG { 19:54:53 DEBUG ----- output ----- 19:54:53 DEBUG { 19:54:53 DEBUG "status": "success", 19:54:53 DEBUG { 19:54:53 DEBUG "status": "success", 19:54:53 DEBUG "data": { 19:54:53 DEBUG "status": "success", 19:54:53 DEBUG "data": { 19:54:53 DEBUG "resultType": "vector", 19:54:53 DEBUG "data": { 19:54:53 DEBUG "resultType": "vector", 19:54:53 DEBUG "result": [ 19:54:53 DEBUG "resultType": "vector", 19:54:53 DEBUG "result": [ 19:54:53 DEBUG { 19:54:53 DEBUG "result": [ 19:54:53 DEBUG { 19:54:53 DEBUG "metric": { 19:54:53 DEBUG { 19:54:53 DEBUG "metric": { 19:54:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:53 DEBUG "metric": { 19:54:53 DEBUG "pod": "ds-idrepo-0" 19:54:53 DEBUG }, 19:54:53 DEBUG }, 19:54:53 DEBUG "pod": "ds-idrepo-0" 19:54:53 DEBUG "value": [ 19:54:53 DEBUG "value": [ 19:54:53 DEBUG }, 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG "value": [ 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG [...] 19:54:53 DEBUG [...] 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG [...] 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG 19:54:53 DEBUG "pod": "ds-idrepo-2" 19:54:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:53 DEBUG 19:54:53 DEBUG }, 19:54:53 DEBUG }, 19:54:53 DEBUG "pod": "ds-idrepo-2" 19:54:53 DEBUG "value": [ 19:54:53 DEBUG "value": [ 19:54:53 DEBUG }, 19:54:53 DEBUG 1713639293, 19:54:53 DEBUG 1713639293, 19:54:53 DEBUG "value": [ 19:54:53 DEBUG "0" 19:54:53 DEBUG "2657.3333333333335" 19:54:53 DEBUG 1713639293, 19:54:53 DEBUG ] 19:54:53 DEBUG ] 19:54:53 DEBUG "0" 19:54:53 DEBUG } 19:54:53 DEBUG } 19:54:53 DEBUG ] 19:54:53 DEBUG ] 19:54:53 DEBUG ] 19:54:53 DEBUG } 19:54:53 DEBUG } 19:54:53 DEBUG } 19:54:53 DEBUG ] 19:54:53 DEBUG } 19:54:53 DEBUG } 19:54:53 DEBUG } 19:54:53 DEBUG } 19:54:54 INFO 19:54:54 INFO [loop_until]: kubectl --namespace=xlou top pods 19:54:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:54:54 INFO [loop_until]: OK (rc = 0) 19:54:54 DEBUG --- stdout --- 19:54:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17284m 8652Mi am-767f6b67b5-qcqz2 16965m 9064Mi am-767f6b67b5-vbltz 18543m 8667Mi ds-cts-0 18905m 10284Mi ds-cts-1 17374m 10272Mi ds-cts-2 18904m 10339Mi ds-idrepo-0 7312m 3888Mi ds-idrepo-1 7321m 3882Mi ds-idrepo-2 7242m 3771Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 14m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 8811m 761Mi 19:54:54 DEBUG --- stderr --- 19:54:54 DEBUG 19:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639294" 19:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639294" 19:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639294" 19:54:54 INFO [http_cmd]: http status code OK 19:54:54 DEBUG --- status code --- 19:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:54 DEBUG --- http response --- 19:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:54 DEBUG ----- output ----- 19:54:54 DEBUG { 19:54:54 DEBUG "status": "success", 19:54:54 DEBUG "data": { 19:54:54 DEBUG "resultType": "vector", 19:54:54 DEBUG "result": [ 19:54:54 DEBUG { 19:54:54 DEBUG "metric": { 19:54:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:54 DEBUG }, 19:54:54 DEBUG "value": [ 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG [...] 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:54 DEBUG }, 19:54:54 DEBUG "value": [ 19:54:54 DEBUG 1713639294, 19:54:54 DEBUG "2669.7" 19:54:54 DEBUG ] 19:54:54 DEBUG } 19:54:54 DEBUG ] 19:54:54 DEBUG } 19:54:54 DEBUG } 19:54:54 INFO [http_cmd]: http status code OK 19:54:54 DEBUG --- status code --- 19:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:54 DEBUG --- http response --- 19:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:54:54 DEBUG ----- output ----- 19:54:54 DEBUG { 19:54:54 DEBUG "status": "success", 19:54:54 DEBUG "data": { 19:54:54 DEBUG "resultType": "vector", 19:54:54 DEBUG "result": [ 19:54:54 DEBUG { 19:54:54 DEBUG "metric": { 19:54:54 DEBUG "pod": "ds-idrepo-1" 19:54:54 DEBUG }, 19:54:54 DEBUG "value": [ 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG [...] 19:54:54 DEBUG 19:54:54 INFO [http_cmd]: http status code OK 19:54:54 DEBUG 19:54:54 DEBUG --- status code --- 19:54:54 DEBUG 19:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:54 DEBUG "pod": "ds-cts-0" 19:54:54 DEBUG --- http response --- 19:54:54 DEBUG }, 19:54:54 DEBUG "value": [ 19:54:54 DEBUG 1713639294, 19:54:54 DEBUG "71847745.26044543" 19:54:54 DEBUG ] 19:54:54 DEBUG } 19:54:54 DEBUG ] 19:54:54 DEBUG } 19:54:54 DEBUG } 19:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:54 DEBUG ----- output ----- 19:54:54 DEBUG { 19:54:54 DEBUG "status": "success", 19:54:54 DEBUG "data": { 19:54:54 DEBUG "resultType": "vector", 19:54:54 DEBUG "result": [ 19:54:54 DEBUG { 19:54:54 DEBUG "metric": { 19:54:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:54:54 DEBUG }, 19:54:54 DEBUG "value": [ 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG [...] 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:54:54 DEBUG }, 19:54:54 DEBUG "value": [ 19:54:54 DEBUG 1713639294, 19:54:54 DEBUG "8.935255071089054" 19:54:54 DEBUG ] 19:54:54 DEBUG } 19:54:54 DEBUG ] 19:54:54 DEBUG } 19:54:54 DEBUG } 19:54:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639294" 19:54:54 INFO [http_cmd]: http status code OK 19:54:54 DEBUG --- status code --- 19:54:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:54 DEBUG --- http response --- 19:54:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:54 DEBUG ----- output ----- 19:54:54 DEBUG { 19:54:54 DEBUG "status": "success", 19:54:54 DEBUG "data": { 19:54:54 DEBUG "resultType": "vector", 19:54:54 DEBUG "result": [ 19:54:54 DEBUG { 19:54:54 DEBUG "metric": { 19:54:54 DEBUG "instance": "10.142.0.107:9100" 19:54:54 DEBUG }, 19:54:54 DEBUG "value": [ 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG [...] 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG 19:54:54 DEBUG "instance": "10.142.0.43:9100" 19:54:54 DEBUG }, 19:54:54 DEBUG "value": [ 19:54:54 DEBUG 1713639294, 19:54:54 DEBUG "3626232060928" 19:54:54 DEBUG ] 19:54:54 DEBUG } 19:54:54 DEBUG ] 19:54:54 DEBUG } 19:54:54 DEBUG } 19:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639295" 19:54:55 INFO [http_cmd]: http status code OK 19:54:55 DEBUG --- status code --- 19:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:55 DEBUG --- http response --- 19:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:55 DEBUG ----- output ----- 19:54:55 DEBUG { 19:54:55 DEBUG "status": "success", 19:54:55 DEBUG "data": { 19:54:55 DEBUG "resultType": "vector", 19:54:55 DEBUG "result": [ 19:54:55 DEBUG { 19:54:55 DEBUG "metric": { 19:54:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:55 DEBUG }, 19:54:55 DEBUG "value": [ 19:54:55 DEBUG 19:54:55 DEBUG 19:54:55 DEBUG 19:54:55 DEBUG [...] 19:54:55 DEBUG 19:54:55 DEBUG 19:54:55 DEBUG 19:54:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:55 DEBUG }, 19:54:55 DEBUG "value": [ 19:54:55 DEBUG 1713639295, 19:54:55 DEBUG "2669.4333333333334" 19:54:55 DEBUG ] 19:54:55 DEBUG } 19:54:55 DEBUG ] 19:54:55 DEBUG } 19:54:55 DEBUG } 19:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639295" 19:54:55 INFO [http_cmd]: http status code OK 19:54:55 DEBUG --- status code --- 19:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:55 DEBUG --- http response --- 19:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:55 DEBUG ----- output ----- 19:54:55 DEBUG { 19:54:55 DEBUG "status": "success", 19:54:55 DEBUG "data": { 19:54:55 DEBUG "resultType": "vector", 19:54:55 DEBUG "result": [ 19:54:55 DEBUG { 19:54:55 DEBUG "metric": { 19:54:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:55 DEBUG }, 19:54:55 DEBUG "value": [ 19:54:55 DEBUG 19:54:55 DEBUG 19:54:55 DEBUG 19:54:55 DEBUG [...] 19:54:55 DEBUG 19:54:55 DEBUG 19:54:55 DEBUG 19:54:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:55 DEBUG }, 19:54:55 DEBUG "value": [ 19:54:55 DEBUG 1713639295, 19:54:55 DEBUG "2669.2" 19:54:55 DEBUG ] 19:54:55 DEBUG } 19:54:55 DEBUG ] 19:54:55 DEBUG } 19:54:55 DEBUG } 19:54:55 INFO 19:54:55 INFO [loop_until]: kubectl --namespace=xlou top node 19:54:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:54:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639295" 19:54:55 INFO [http_cmd]: http status code OK 19:54:55 DEBUG --- status code --- 19:54:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:55 DEBUG --- http response --- 19:54:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:55 DEBUG ----- output ----- 19:54:55 DEBUG { 19:54:55 DEBUG "status": "success", 19:54:55 DEBUG "data": { 19:54:55 DEBUG "resultType": "vector", 19:54:55 DEBUG "result": [ 19:54:55 DEBUG { 19:54:55 DEBUG "metric": { 19:54:55 DEBUG "pod": "ds-idrepo-1" 19:54:55 DEBUG }, 19:54:55 DEBUG "value": [ 19:54:55 DEBUG 19:54:55 DEBUG 19:54:55 DEBUG 19:54:55 DEBUG [...] 19:54:55 DEBUG 19:54:55 DEBUG 19:54:55 DEBUG 19:54:55 DEBUG "pod": "ds-cts-0" 19:54:55 DEBUG }, 19:54:55 DEBUG "value": [ 19:54:55 DEBUG 1713639295, 19:54:55 DEBUG "19.17849389533108" 19:54:55 DEBUG ] 19:54:55 DEBUG } 19:54:55 DEBUG ] 19:54:55 DEBUG } 19:54:55 DEBUG } 19:54:55 INFO [loop_until]: OK (rc = 0) 19:54:55 DEBUG --- stdout --- 19:54:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6211Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7533m 25% 7639Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16990m 56% 11373Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3345Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18966m 63% 13465Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5225Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17313m 57% 11952Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14506m 48% 2962Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19384m 64% 10889Mi 9% gke-xlou-cdm-ds-0f601751-j488 8167m 27% 7171Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16927m 56% 15326Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8110m 27% 7858Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18249m 61% 13008Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9116m 57% 6370Mi 10% 19:54:55 DEBUG --- stderr --- 19:54:55 DEBUG 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639296" 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG { 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG "data": { 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG "result": [ 19:54:56 DEBUG { 19:54:56 DEBUG "metric": { 19:54:56 DEBUG "pod": "ds-idrepo-0" 19:54:56 DEBUG }, 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG [...] 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG "pod": "ds-idrepo-2" 19:54:56 DEBUG }, 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG "0" 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639296" 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG { 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG "data": { 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG "result": [ 19:54:56 DEBUG { 19:54:56 DEBUG "metric": { 19:54:56 DEBUG "pod": "ds-cts-0" 19:54:56 DEBUG }, 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG [...] 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG "pod": "ds-cts-2" 19:54:56 DEBUG }, 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG "0" 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639296" 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639296" 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639296" 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639296" 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639296" 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639296" 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639296" 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639296" 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639296" 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639296" 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639296" 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639296" 19:54:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639296" 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 DEBUG --- status code --- 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 DEBUG --- status code --- 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 INFO [http_cmd]: http status code OK 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG --- status code --- 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG { 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG { 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG { 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG { 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG { 19:54:56 DEBUG { 19:54:56 DEBUG { 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG { 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG --- http response --- 19:54:56 DEBUG { 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG "data": { 19:54:56 DEBUG { 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG "data": { 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG "data": { 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG "data": { 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG "data": { 19:54:56 DEBUG "data": { 19:54:56 DEBUG "data": { 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG "data": { 19:54:56 DEBUG "result": [ 19:54:56 DEBUG { 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG "data": { 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG "result": [ 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG "data": { 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG "result": [ 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG { 19:54:56 DEBUG "data": { 19:54:56 DEBUG ----- output ----- 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG "result": [ 19:54:56 DEBUG { 19:54:56 DEBUG { 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG "result": [ 19:54:56 DEBUG "result": [ 19:54:56 DEBUG "result": [ 19:54:56 DEBUG { 19:54:56 DEBUG "result": [ 19:54:56 DEBUG "metric": { 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG { 19:54:56 DEBUG "result": [ 19:54:56 DEBUG { 19:54:56 DEBUG "metric": { 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG "result": [ 19:54:56 DEBUG { 19:54:56 DEBUG { 19:54:56 DEBUG { 19:54:56 DEBUG "metric": { 19:54:56 DEBUG { 19:54:56 DEBUG "instance": "10.142.0.107:9100" 19:54:56 DEBUG "result": [ 19:54:56 DEBUG "status": "success", 19:54:56 DEBUG { 19:54:56 DEBUG "metric": { 19:54:56 DEBUG "pod": "ds-idrepo-1" 19:54:56 DEBUG "data": { 19:54:56 DEBUG { 19:54:56 DEBUG "metric": { 19:54:56 DEBUG "metric": { 19:54:56 DEBUG "metric": { 19:54:56 DEBUG "pod": "ds-cts-0" 19:54:56 DEBUG "metric": { 19:54:56 DEBUG }, 19:54:56 DEBUG { 19:54:56 DEBUG "data": { 19:54:56 DEBUG "metric": { 19:54:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:56 DEBUG }, 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG "metric": { 19:54:56 DEBUG "pod": "ds-idrepo-0" 19:54:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:54:56 DEBUG }, 19:54:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:54:56 DEBUG "value": [ 19:54:56 DEBUG "metric": { 19:54:56 DEBUG "resultType": "vector", 19:54:56 DEBUG "result": [ 19:54:56 DEBUG }, 19:54:56 DEBUG "value": [ 19:54:56 DEBUG "result": [ 19:54:56 DEBUG "instance": "10.142.0.107:9100" 19:54:56 DEBUG }, 19:54:56 DEBUG }, 19:54:56 DEBUG }, 19:54:56 DEBUG "value": [ 19:54:56 DEBUG }, 19:54:56 DEBUG 19:54:56 DEBUG "instance": "10.142.0.107:9100" 19:54:56 DEBUG }, 19:54:56 DEBUG { 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 19:54:56 DEBUG { 19:54:56 DEBUG "metric": { 19:54:56 DEBUG "value": [ 19:54:56 DEBUG "value": [ 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 19:54:56 DEBUG "value": [ 19:54:56 DEBUG "pod": "ds-cts-0" 19:54:56 DEBUG "metric": { 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG "instance": "10.142.0.107:9100" 19:54:56 DEBUG }, 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG }, 19:54:56 DEBUG "pod": "ds-idrepo-1" 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG }, 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG }, 19:54:56 DEBUG 19:54:56 DEBUG [...] 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG [...] 19:54:56 DEBUG [...] 19:54:56 DEBUG [...] 19:54:56 DEBUG 19:54:56 DEBUG "value": [ 19:54:56 DEBUG "value": [ 19:54:56 DEBUG [...] 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG [...] 19:54:56 DEBUG [...] 19:54:56 DEBUG [...] 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG [...] 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG "pod": "ds-cts-0" 19:54:56 DEBUG 19:54:56 DEBUG [...] 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG "instance": "10.142.0.43:9100" 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:56 DEBUG }, 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG "pod": "ds-cts-2" 19:54:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:56 DEBUG 19:54:56 DEBUG }, 19:54:56 DEBUG [...] 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG "value": [ 19:54:56 DEBUG }, 19:54:56 DEBUG 19:54:56 DEBUG "pod": "ds-cts-2" 19:54:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:54:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:54:56 DEBUG }, 19:54:56 DEBUG }, 19:54:56 DEBUG "instance": "10.142.0.43:9100" 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 19:54:56 DEBUG [...] 19:54:56 DEBUG [...] 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG "0" 19:54:56 DEBUG 19:54:56 DEBUG }, 19:54:56 DEBUG }, 19:54:56 DEBUG }, 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG }, 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG ] 19:54:56 DEBUG "value": [ 19:54:56 DEBUG "instance": "10.142.0.43:9100" 19:54:56 DEBUG "value": [ 19:54:56 DEBUG "value": [ 19:54:56 DEBUG "value": [ 19:54:56 DEBUG "29.966666666666665" 19:54:56 DEBUG "value": [ 19:54:56 DEBUG "80790022.26666667" 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG "330635206.5770906" 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG ] 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG ] 19:54:56 DEBUG "80784961.6" 19:54:56 DEBUG "pod": "ds-cts-2" 19:54:56 DEBUG 19:54:56 DEBUG "instance": "10.142.0.43:9100" 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG }, 19:54:56 DEBUG "value": [ 19:54:56 DEBUG "0.156" 19:54:56 DEBUG "896999424" 19:54:56 DEBUG } 19:54:56 DEBUG "480217.5" 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG }, 19:54:56 DEBUG "pod": "ds-cts-0" 19:54:56 DEBUG }, 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG "0" 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG ] 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG "value": [ 19:54:56 DEBUG }, 19:54:56 DEBUG "value": [ 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG ] 19:54:56 DEBUG "0.19420000000001683" 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG "value": [ 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG "0.0002077444444444444" 19:54:56 DEBUG 1713639296, 19:54:56 DEBUG "60785353.70268115" 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG "0.07499999999999432" 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG ] 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG } 19:54:56 DEBUG } 19:55:39 WARNING File /tmp/lodemon_data-20240420_191838/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [20/Apr/2024 19:55:39] "GET /monitoring/average?start_time=20240420_182152&stop_time=20240420_185338 HTTP/1.1" 200 - 19:55:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639352" 19:55:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639352" 19:55:52 ERROR [http_cmd]: ERROR 19:55:52 ERROR --- status code --- 19:55:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:55:52 ERROR --- http response --- 19:55:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:55:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:55:52 INFO [http_cmd]: http status code OK 19:55:52 DEBUG --- status code --- 19:55:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:52 DEBUG --- http response --- 19:55:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:55:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639353" 19:55:53 INFO [http_cmd]: http status code OK 19:55:53 DEBUG --- status code --- 19:55:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:53 DEBUG --- http response --- 19:55:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:53 DEBUG ----- output ----- 19:55:53 DEBUG { 19:55:53 DEBUG "status": "success", 19:55:53 DEBUG "data": { 19:55:53 DEBUG "resultType": "vector", 19:55:53 DEBUG "result": [ 19:55:53 DEBUG { 19:55:53 DEBUG "metric": { 19:55:53 DEBUG "pod": "ds-idrepo-0" 19:55:53 DEBUG }, 19:55:53 DEBUG "value": [ 19:55:53 DEBUG 19:55:53 DEBUG 19:55:53 DEBUG 19:55:53 DEBUG [...] 19:55:53 DEBUG 19:55:53 DEBUG 19:55:53 DEBUG 19:55:53 DEBUG "pod": "ds-idrepo-2" 19:55:53 DEBUG }, 19:55:53 DEBUG "value": [ 19:55:53 DEBUG 1713639353, 19:55:53 DEBUG "0" 19:55:53 DEBUG ] 19:55:53 DEBUG } 19:55:53 DEBUG ] 19:55:53 DEBUG } 19:55:53 DEBUG } 19:55:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639353" 19:55:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639353" 19:55:53 INFO [http_cmd]: http status code OK 19:55:53 DEBUG --- status code --- 19:55:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:53 DEBUG --- http response --- 19:55:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:53 DEBUG ----- output ----- 19:55:53 DEBUG { 19:55:53 INFO [http_cmd]: http status code OK 19:55:53 DEBUG "status": "success", 19:55:53 DEBUG --- status code --- 19:55:53 DEBUG "data": { 19:55:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:53 DEBUG "resultType": "vector", 19:55:53 DEBUG --- http response --- 19:55:53 DEBUG "result": [ 19:55:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:53 DEBUG { 19:55:53 DEBUG ----- output ----- 19:55:53 DEBUG "metric": { 19:55:53 DEBUG { 19:55:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:53 DEBUG "status": "success", 19:55:53 DEBUG }, 19:55:53 DEBUG "data": { 19:55:53 DEBUG "value": [ 19:55:53 DEBUG "resultType": "vector", 19:55:53 DEBUG 19:55:53 DEBUG "result": [ 19:55:53 DEBUG 19:55:53 DEBUG { 19:55:53 DEBUG 19:55:53 DEBUG "metric": { 19:55:53 DEBUG [...] 19:55:53 DEBUG "pod": "ds-idrepo-0" 19:55:53 DEBUG 19:55:53 DEBUG }, 19:55:53 DEBUG 19:55:53 DEBUG "value": [ 19:55:53 DEBUG 19:55:53 DEBUG 19:55:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:53 DEBUG 19:55:53 DEBUG }, 19:55:53 DEBUG 19:55:53 DEBUG "value": [ 19:55:53 DEBUG [...] 19:55:53 DEBUG 1713639353, 19:55:53 DEBUG 19:55:53 DEBUG "2673.1" 19:55:53 DEBUG 19:55:53 DEBUG ] 19:55:53 DEBUG 19:55:53 DEBUG } 19:55:53 DEBUG "pod": "ds-idrepo-2" 19:55:53 DEBUG ] 19:55:53 DEBUG }, 19:55:53 DEBUG } 19:55:53 DEBUG "value": [ 19:55:53 DEBUG } 19:55:53 DEBUG 1713639353, 19:55:53 DEBUG "0" 19:55:53 DEBUG ] 19:55:53 DEBUG } 19:55:53 DEBUG ] 19:55:53 DEBUG } 19:55:53 DEBUG } 19:55:54 INFO 19:55:54 INFO [loop_until]: kubectl --namespace=xlou top pods 19:55:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:55:54 INFO [loop_until]: OK (rc = 0) 19:55:54 DEBUG --- stdout --- 19:55:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15098m 8648Mi am-767f6b67b5-qcqz2 18009m 9064Mi am-767f6b67b5-vbltz 14333m 8667Mi ds-cts-0 15671m 10296Mi ds-cts-1 17114m 10324Mi ds-cts-2 18886m 10346Mi ds-idrepo-0 7524m 3888Mi ds-idrepo-1 7113m 3882Mi ds-idrepo-2 7345m 3771Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 12m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 6896m 188Mi 19:55:54 DEBUG --- stderr --- 19:55:54 DEBUG 19:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639354" 19:55:54 INFO [http_cmd]: http status code OK 19:55:54 DEBUG --- status code --- 19:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:54 DEBUG --- http response --- 19:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:55:54 DEBUG ----- output ----- 19:55:54 DEBUG { 19:55:54 DEBUG "status": "success", 19:55:54 DEBUG "data": { 19:55:54 DEBUG "resultType": "vector", 19:55:54 DEBUG "result": [ 19:55:54 DEBUG { 19:55:54 DEBUG "metric": { 19:55:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:55:54 DEBUG }, 19:55:54 DEBUG "value": [ 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG [...] 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:55:54 DEBUG }, 19:55:54 DEBUG "value": [ 19:55:54 DEBUG 1713639354, 19:55:54 DEBUG "370062.14536611724" 19:55:54 DEBUG ] 19:55:54 DEBUG } 19:55:54 DEBUG ] 19:55:54 DEBUG } 19:55:54 DEBUG } 19:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639354" 19:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639354" 19:55:54 INFO [http_cmd]: http status code OK 19:55:54 DEBUG --- status code --- 19:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:54 DEBUG --- http response --- 19:55:54 INFO [http_cmd]: http status code OK 19:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:54 DEBUG --- status code --- 19:55:54 DEBUG ----- output ----- 19:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:54 DEBUG { 19:55:54 DEBUG --- http response --- 19:55:54 DEBUG "status": "success", 19:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:54 DEBUG "data": { 19:55:54 DEBUG ----- output ----- 19:55:54 DEBUG "resultType": "vector", 19:55:54 DEBUG { 19:55:54 DEBUG "result": [ 19:55:54 DEBUG "status": "success", 19:55:54 DEBUG { 19:55:54 DEBUG "data": { 19:55:54 DEBUG "metric": { 19:55:54 DEBUG "resultType": "vector", 19:55:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:55:54 DEBUG "result": [ 19:55:54 DEBUG }, 19:55:54 DEBUG { 19:55:54 DEBUG "value": [ 19:55:54 DEBUG "metric": { 19:55:54 DEBUG 19:55:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:54 DEBUG 19:55:54 DEBUG }, 19:55:54 DEBUG 19:55:54 DEBUG "value": [ 19:55:54 DEBUG [...] 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG [...] 19:55:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:55:54 DEBUG 19:55:54 DEBUG }, 19:55:54 DEBUG 19:55:54 DEBUG "value": [ 19:55:54 DEBUG 19:55:54 DEBUG 1713639354, 19:55:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:54 DEBUG "7.007876655857802" 19:55:54 DEBUG }, 19:55:54 DEBUG ] 19:55:54 DEBUG "value": [ 19:55:54 DEBUG } 19:55:54 DEBUG 1713639354, 19:55:54 DEBUG ] 19:55:54 DEBUG "4.9" 19:55:54 DEBUG } 19:55:54 DEBUG ] 19:55:54 DEBUG } 19:55:54 DEBUG } 19:55:54 DEBUG ] 19:55:54 DEBUG } 19:55:54 DEBUG } 19:55:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639354" 19:55:54 INFO [http_cmd]: http status code OK 19:55:54 DEBUG --- status code --- 19:55:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:54 DEBUG --- http response --- 19:55:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:54 DEBUG ----- output ----- 19:55:54 DEBUG { 19:55:54 DEBUG "status": "success", 19:55:54 DEBUG "data": { 19:55:54 DEBUG "resultType": "vector", 19:55:54 DEBUG "result": [ 19:55:54 DEBUG { 19:55:54 DEBUG "metric": { 19:55:54 DEBUG "instance": "10.142.0.107:9100" 19:55:54 DEBUG }, 19:55:54 DEBUG "value": [ 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG [...] 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG 19:55:54 DEBUG "instance": "10.142.0.43:9100" 19:55:54 DEBUG }, 19:55:54 DEBUG "value": [ 19:55:54 DEBUG 1713639354, 19:55:54 DEBUG "3629853113344" 19:55:54 DEBUG ] 19:55:54 DEBUG } 19:55:54 DEBUG ] 19:55:54 DEBUG } 19:55:54 DEBUG } 19:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639355" 19:55:55 INFO [http_cmd]: http status code OK 19:55:55 DEBUG --- status code --- 19:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:55 DEBUG --- http response --- 19:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:55 DEBUG ----- output ----- 19:55:55 DEBUG { 19:55:55 DEBUG "status": "success", 19:55:55 DEBUG "data": { 19:55:55 DEBUG "resultType": "vector", 19:55:55 DEBUG "result": [ 19:55:55 DEBUG { 19:55:55 DEBUG "metric": { 19:55:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:55 DEBUG }, 19:55:55 DEBUG "value": [ 19:55:55 DEBUG 19:55:55 DEBUG 19:55:55 DEBUG 19:55:55 DEBUG [...] 19:55:55 DEBUG 19:55:55 DEBUG 19:55:55 DEBUG 19:55:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:55 DEBUG }, 19:55:55 DEBUG "value": [ 19:55:55 DEBUG 1713639355, 19:55:55 DEBUG "4.966666666666667" 19:55:55 DEBUG ] 19:55:55 DEBUG } 19:55:55 DEBUG ] 19:55:55 DEBUG } 19:55:55 DEBUG } 19:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639355" 19:55:55 INFO [http_cmd]: http status code OK 19:55:55 DEBUG --- status code --- 19:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:55 DEBUG --- http response --- 19:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:55 DEBUG ----- output ----- 19:55:55 DEBUG { 19:55:55 DEBUG "status": "success", 19:55:55 DEBUG "data": { 19:55:55 DEBUG "resultType": "vector", 19:55:55 DEBUG "result": [ 19:55:55 DEBUG { 19:55:55 DEBUG "metric": { 19:55:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:55 DEBUG }, 19:55:55 DEBUG "value": [ 19:55:55 DEBUG 19:55:55 DEBUG 19:55:55 DEBUG 19:55:55 DEBUG [...] 19:55:55 DEBUG 19:55:55 DEBUG 19:55:55 DEBUG 19:55:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:55 DEBUG }, 19:55:55 DEBUG "value": [ 19:55:55 DEBUG 1713639355, 19:55:55 DEBUG "4.966666666666667" 19:55:55 DEBUG ] 19:55:55 DEBUG } 19:55:55 DEBUG ] 19:55:55 DEBUG } 19:55:55 DEBUG } 19:55:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639355" 19:55:55 INFO [http_cmd]: http status code OK 19:55:55 DEBUG --- status code --- 19:55:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:55 DEBUG --- http response --- 19:55:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:55:55 DEBUG ----- output ----- 19:55:55 DEBUG { 19:55:55 DEBUG "status": "success", 19:55:55 DEBUG "data": { 19:55:55 DEBUG "resultType": "vector", 19:55:55 DEBUG "result": [ 19:55:55 DEBUG { 19:55:55 DEBUG "metric": { 19:55:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:55:55 DEBUG }, 19:55:55 DEBUG "value": [ 19:55:55 DEBUG 19:55:55 DEBUG 19:55:55 DEBUG 19:55:55 DEBUG [...] 19:55:55 DEBUG 19:55:55 DEBUG 19:55:55 DEBUG 19:55:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:55:55 DEBUG }, 19:55:55 DEBUG "value": [ 19:55:55 DEBUG 1713639355, 19:55:55 DEBUG "6.467712591259121" 19:55:55 DEBUG ] 19:55:55 DEBUG } 19:55:55 DEBUG ] 19:55:55 DEBUG } 19:55:55 DEBUG } 19:55:55 INFO 19:55:55 INFO [loop_until]: kubectl --namespace=xlou top node 19:55:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:55:55 INFO [loop_until]: OK (rc = 0) 19:55:55 DEBUG --- stdout --- 19:55:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6218Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6487m 21% 7629Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14485m 48% 11377Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3339Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18131m 60% 13481Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5214Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 12475m 41% 11960Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 14540m 48% 2966Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 17551m 58% 10901Mi 9% gke-xlou-cdm-ds-0f601751-j488 5784m 19% 7168Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16447m 55% 15372Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 6537m 21% 7861Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 16941m 56% 13011Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 6323m 39% 5806Mi 9% 19:55:55 DEBUG --- stderr --- 19:55:55 DEBUG 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639356" 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 DEBUG --- status code --- 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG { 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG "data": { 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "result": [ 19:55:56 DEBUG { 19:55:56 DEBUG "metric": { 19:55:56 DEBUG "pod": "ds-idrepo-0" 19:55:56 DEBUG }, 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG "pod": "ds-idrepo-2" 19:55:56 DEBUG }, 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG "0" 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639356" 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 DEBUG --- status code --- 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG { 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG "data": { 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "result": [ 19:55:56 DEBUG { 19:55:56 DEBUG "metric": { 19:55:56 DEBUG "pod": "ds-cts-0" 19:55:56 DEBUG }, 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG "pod": "ds-cts-2" 19:55:56 DEBUG }, 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG "0" 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639356" 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639356" 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639356" 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639356" 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639356" 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639356" 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639356" 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639356" 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639356" 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639356" 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639356" 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639356" 19:55:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639356" 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 DEBUG --- status code --- 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 DEBUG --- status code --- 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 DEBUG --- status code --- 19:55:56 DEBUG --- status code --- 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG --- status code --- 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 DEBUG --- status code --- 19:55:56 DEBUG --- status code --- 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG --- status code --- 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG --- status code --- 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG --- status code --- 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 DEBUG --- status code --- 19:55:56 DEBUG --- status code --- 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:56 INFO [http_cmd]: http status code OK 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG { 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG --- status code --- 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG { 19:55:56 DEBUG { 19:55:56 DEBUG { 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG { 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG { 19:55:56 DEBUG { 19:55:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG { 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG "data": { 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG --- http response --- 19:55:56 DEBUG { 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG "data": { 19:55:56 DEBUG "data": { 19:55:56 DEBUG "data": { 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "data": { 19:55:56 DEBUG { 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG "data": { 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG "data": { 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:55:56 DEBUG "data": { 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "result": [ 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG { 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG { 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "data": { 19:55:56 DEBUG ----- output ----- 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "result": [ 19:55:56 DEBUG "result": [ 19:55:56 DEBUG "result": [ 19:55:56 DEBUG { 19:55:56 DEBUG "result": [ 19:55:56 DEBUG "data": { 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG "result": [ 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG "result": [ 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG { 19:55:56 DEBUG "result": [ 19:55:56 DEBUG { 19:55:56 DEBUG { 19:55:56 DEBUG { 19:55:56 DEBUG "metric": { 19:55:56 DEBUG { 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "data": { 19:55:56 DEBUG { 19:55:56 DEBUG "data": { 19:55:56 DEBUG { 19:55:56 DEBUG "result": [ 19:55:56 DEBUG "status": "success", 19:55:56 DEBUG { 19:55:56 DEBUG "metric": { 19:55:56 DEBUG "metric": { 19:55:56 DEBUG "metric": { 19:55:56 DEBUG "pod": "ds-idrepo-0" 19:55:56 DEBUG "metric": { 19:55:56 DEBUG "result": [ 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "metric": { 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "metric": { 19:55:56 DEBUG { 19:55:56 DEBUG "data": { 19:55:56 DEBUG "metric": { 19:55:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:56 DEBUG "pod": "ds-cts-0" 19:55:56 DEBUG "pod": "ds-cts-0" 19:55:56 DEBUG }, 19:55:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:56 DEBUG { 19:55:56 DEBUG "result": [ 19:55:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:55:56 DEBUG "result": [ 19:55:56 DEBUG "instance": "10.142.0.107:9100" 19:55:56 DEBUG "metric": { 19:55:56 DEBUG "resultType": "vector", 19:55:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:55:56 DEBUG }, 19:55:56 DEBUG }, 19:55:56 DEBUG }, 19:55:56 DEBUG "value": [ 19:55:56 DEBUG }, 19:55:56 DEBUG "metric": { 19:55:56 DEBUG { 19:55:56 DEBUG }, 19:55:56 DEBUG { 19:55:56 DEBUG }, 19:55:56 DEBUG "instance": "10.142.0.107:9100" 19:55:56 DEBUG "result": [ 19:55:56 DEBUG }, 19:55:56 DEBUG "value": [ 19:55:56 DEBUG "value": [ 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 19:55:56 DEBUG "value": [ 19:55:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:55:56 DEBUG "metric": { 19:55:56 DEBUG "value": [ 19:55:56 DEBUG "metric": { 19:55:56 DEBUG "value": [ 19:55:56 DEBUG }, 19:55:56 DEBUG { 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG }, 19:55:56 DEBUG "instance": "10.142.0.107:9100" 19:55:56 DEBUG 19:55:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:55:56 DEBUG 19:55:56 DEBUG "value": [ 19:55:56 DEBUG "metric": { 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG "value": [ 19:55:56 DEBUG }, 19:55:56 DEBUG 19:55:56 DEBUG }, 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG "instance": "10.142.0.107:9100" 19:55:56 DEBUG }, 19:55:56 DEBUG 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG [...] 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG "pod": "ds-cts-2" 19:55:56 DEBUG [...] 19:55:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG [...] 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG "pod": "ds-cts-2" 19:55:56 DEBUG }, 19:55:56 DEBUG 19:55:56 DEBUG }, 19:55:56 DEBUG 19:55:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:55:56 DEBUG 19:55:56 DEBUG "instance": "10.142.0.43:9100" 19:55:56 DEBUG 19:55:56 DEBUG 19:55:56 DEBUG }, 19:55:56 DEBUG 19:55:56 DEBUG "pod": "ds-cts-2" 19:55:56 DEBUG }, 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 19:55:56 DEBUG }, 19:55:56 DEBUG 19:55:56 DEBUG }, 19:55:56 DEBUG "instance": "10.142.0.43:9100" 19:55:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 19:55:56 DEBUG }, 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 19:55:56 DEBUG "value": [ 19:55:56 DEBUG }, 19:55:56 DEBUG }, 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG "instance": "10.142.0.43:9100" 19:55:56 DEBUG "value": [ 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG "29.933333333333334" 19:55:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:55:56 DEBUG "0.156" 19:55:56 DEBUG "instance": "10.142.0.43:9100" 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG "pod": "idm-b85f9d978-wnhxb" 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG "value": [ 19:55:56 DEBUG "value": [ 19:55:56 DEBUG "0" 19:55:56 DEBUG }, 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG "0" 19:55:56 DEBUG ] 19:55:56 DEBUG }, 19:55:56 DEBUG ] 19:55:56 DEBUG }, 19:55:56 DEBUG "0" 19:55:56 DEBUG }, 19:55:56 DEBUG "0.18870000000002846" 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG ] 19:55:56 DEBUG "value": [ 19:55:56 DEBUG "0.000018215555555555553" 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG "value": [ 19:55:56 DEBUG } 19:55:56 DEBUG "value": [ 19:55:56 DEBUG ] 19:55:56 DEBUG "value": [ 19:55:56 DEBUG ] 19:55:56 DEBUG "35065633.766666666" 19:55:56 DEBUG "292847616" 19:55:56 DEBUG } 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG ] 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG } 19:55:56 DEBUG 1713639356, 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG ] 19:55:56 DEBUG ] 19:55:56 DEBUG "0.0863333333333325" 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG "429020.5" 19:55:56 DEBUG } 19:55:56 DEBUG "35060899.233333334" 19:55:56 DEBUG ] 19:55:56 DEBUG "0" 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG ] 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG ] 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:55:56 DEBUG } 19:56:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639412" 19:56:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639412" 19:56:52 ERROR [http_cmd]: ERROR 19:56:52 ERROR --- status code --- 19:56:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:56:52 ERROR --- http response --- 19:56:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:56:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:56:52 INFO [http_cmd]: http status code OK 19:56:52 DEBUG --- status code --- 19:56:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:52 DEBUG --- http response --- 19:56:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:56:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639413" 19:56:53 INFO [http_cmd]: http status code OK 19:56:53 DEBUG --- status code --- 19:56:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:53 DEBUG --- http response --- 19:56:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:53 DEBUG ----- output ----- 19:56:53 DEBUG { 19:56:53 DEBUG "status": "success", 19:56:53 DEBUG "data": { 19:56:53 DEBUG "resultType": "vector", 19:56:53 DEBUG "result": [ 19:56:53 DEBUG { 19:56:53 DEBUG "metric": { 19:56:53 DEBUG "pod": "ds-idrepo-0" 19:56:53 DEBUG }, 19:56:53 DEBUG "value": [ 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG [...] 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG "pod": "ds-idrepo-2" 19:56:53 DEBUG }, 19:56:53 DEBUG "value": [ 19:56:53 DEBUG 1713639413, 19:56:53 DEBUG "0" 19:56:53 DEBUG ] 19:56:53 DEBUG } 19:56:53 DEBUG ] 19:56:53 DEBUG } 19:56:53 DEBUG } 19:56:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639413" 19:56:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639413" 19:56:53 INFO [http_cmd]: http status code OK 19:56:53 DEBUG --- status code --- 19:56:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:53 INFO [http_cmd]: http status code OK 19:56:53 DEBUG --- http response --- 19:56:53 DEBUG --- status code --- 19:56:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:53 DEBUG --- http response --- 19:56:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:53 DEBUG ----- output ----- 19:56:53 DEBUG ----- output ----- 19:56:53 DEBUG { 19:56:53 DEBUG { 19:56:53 DEBUG "status": "success", 19:56:53 DEBUG "status": "success", 19:56:53 DEBUG "data": { 19:56:53 DEBUG "data": { 19:56:53 DEBUG "resultType": "vector", 19:56:53 DEBUG "resultType": "vector", 19:56:53 DEBUG "result": [ 19:56:53 DEBUG "result": [ 19:56:53 DEBUG { 19:56:53 DEBUG { 19:56:53 DEBUG "metric": { 19:56:53 DEBUG "metric": { 19:56:53 DEBUG "pod": "ds-idrepo-0" 19:56:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:53 DEBUG }, 19:56:53 DEBUG }, 19:56:53 DEBUG "value": [ 19:56:53 DEBUG "value": [ 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG [...] 19:56:53 DEBUG [...] 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG 19:56:53 DEBUG "pod": "ds-idrepo-2" 19:56:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:53 DEBUG }, 19:56:53 DEBUG }, 19:56:53 DEBUG "value": [ 19:56:53 DEBUG "value": [ 19:56:53 DEBUG 1713639413, 19:56:53 DEBUG 1713639413, 19:56:53 DEBUG "0" 19:56:53 DEBUG "2671.7" 19:56:53 DEBUG ] 19:56:53 DEBUG ] 19:56:53 DEBUG } 19:56:53 DEBUG } 19:56:53 DEBUG ] 19:56:53 DEBUG ] 19:56:53 DEBUG } 19:56:53 DEBUG } 19:56:53 DEBUG } 19:56:53 DEBUG } 19:56:54 INFO 19:56:54 INFO [loop_until]: kubectl --namespace=xlou top pods 19:56:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639414" 19:56:54 INFO [http_cmd]: http status code OK 19:56:54 DEBUG --- status code --- 19:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:54 DEBUG --- http response --- 19:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:56:54 DEBUG ----- output ----- 19:56:54 DEBUG { 19:56:54 DEBUG "status": "success", 19:56:54 DEBUG "data": { 19:56:54 DEBUG "resultType": "vector", 19:56:54 DEBUG "result": [ 19:56:54 DEBUG { 19:56:54 DEBUG "metric": { 19:56:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:56:54 DEBUG }, 19:56:54 DEBUG "value": [ 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG [...] 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:56:54 DEBUG }, 19:56:54 DEBUG "value": [ 19:56:54 DEBUG 1713639414, 19:56:54 DEBUG "0" 19:56:54 DEBUG ] 19:56:54 DEBUG } 19:56:54 DEBUG ] 19:56:54 DEBUG } 19:56:54 DEBUG } 19:56:54 INFO [loop_until]: OK (rc = 0) 19:56:54 DEBUG --- stdout --- 19:56:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 9387m 8651Mi am-767f6b67b5-qcqz2 8896m 9062Mi am-767f6b67b5-vbltz 12980m 8667Mi ds-cts-0 15722m 10344Mi ds-cts-1 12981m 10321Mi ds-cts-2 14240m 10404Mi ds-idrepo-0 4277m 3893Mi ds-idrepo-1 2511m 3882Mi ds-idrepo-2 4235m 3771Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 5m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 13m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 5725m 739Mi 19:56:54 DEBUG --- stderr --- 19:56:54 DEBUG 19:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639414" 19:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639414" 19:56:54 INFO [http_cmd]: http status code OK 19:56:54 DEBUG --- status code --- 19:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:54 DEBUG --- http response --- 19:56:54 INFO [http_cmd]: http status code OK 19:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:54 DEBUG --- status code --- 19:56:54 DEBUG ----- output ----- 19:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:54 DEBUG { 19:56:54 DEBUG --- http response --- 19:56:54 DEBUG "status": "success", 19:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:54 DEBUG "data": { 19:56:54 DEBUG ----- output ----- 19:56:54 DEBUG "resultType": "vector", 19:56:54 DEBUG { 19:56:54 DEBUG "result": [ 19:56:54 DEBUG "status": "success", 19:56:54 DEBUG { 19:56:54 DEBUG "data": { 19:56:54 DEBUG "metric": { 19:56:54 DEBUG "resultType": "vector", 19:56:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:54 DEBUG "result": [ 19:56:54 DEBUG }, 19:56:54 DEBUG { 19:56:54 DEBUG "value": [ 19:56:54 DEBUG "metric": { 19:56:54 DEBUG 19:56:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:56:54 DEBUG 19:56:54 DEBUG }, 19:56:54 DEBUG 19:56:54 DEBUG "value": [ 19:56:54 DEBUG [...] 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG [...] 19:56:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:54 DEBUG 19:56:54 DEBUG }, 19:56:54 DEBUG 19:56:54 DEBUG "value": [ 19:56:54 DEBUG 19:56:54 DEBUG 1713639414, 19:56:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:56:54 DEBUG "2819.1" 19:56:54 DEBUG }, 19:56:54 DEBUG ] 19:56:54 DEBUG "value": [ 19:56:54 DEBUG } 19:56:54 DEBUG 1713639414, 19:56:54 DEBUG ] 19:56:54 DEBUG "5.763480835797968" 19:56:54 DEBUG } 19:56:54 DEBUG ] 19:56:54 DEBUG } 19:56:54 DEBUG } 19:56:54 DEBUG ] 19:56:54 DEBUG } 19:56:54 DEBUG } 19:56:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639414" 19:56:54 INFO [http_cmd]: http status code OK 19:56:54 DEBUG --- status code --- 19:56:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:54 DEBUG --- http response --- 19:56:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:54 DEBUG ----- output ----- 19:56:54 DEBUG { 19:56:54 DEBUG "status": "success", 19:56:54 DEBUG "data": { 19:56:54 DEBUG "resultType": "vector", 19:56:54 DEBUG "result": [ 19:56:54 DEBUG { 19:56:54 DEBUG "metric": { 19:56:54 DEBUG "instance": "10.142.0.107:9100" 19:56:54 DEBUG }, 19:56:54 DEBUG "value": [ 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG [...] 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG 19:56:54 DEBUG "instance": "10.142.0.43:9100" 19:56:54 DEBUG }, 19:56:54 DEBUG "value": [ 19:56:54 DEBUG 1713639414, 19:56:54 DEBUG "3632104954880" 19:56:54 DEBUG ] 19:56:54 DEBUG } 19:56:54 DEBUG ] 19:56:54 DEBUG } 19:56:54 DEBUG } 19:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639415" 19:56:55 INFO [http_cmd]: http status code OK 19:56:55 DEBUG --- status code --- 19:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:55 DEBUG --- http response --- 19:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:55 DEBUG ----- output ----- 19:56:55 DEBUG { 19:56:55 DEBUG "status": "success", 19:56:55 DEBUG "data": { 19:56:55 DEBUG "resultType": "vector", 19:56:55 DEBUG "result": [ 19:56:55 DEBUG { 19:56:55 DEBUG "metric": { 19:56:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:55 DEBUG }, 19:56:55 DEBUG "value": [ 19:56:55 DEBUG 19:56:55 DEBUG 19:56:55 DEBUG 19:56:55 DEBUG [...] 19:56:55 DEBUG 19:56:55 DEBUG 19:56:55 DEBUG 19:56:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:55 DEBUG }, 19:56:55 DEBUG "value": [ 19:56:55 DEBUG 1713639415, 19:56:55 DEBUG "2817.8" 19:56:55 DEBUG ] 19:56:55 DEBUG } 19:56:55 DEBUG ] 19:56:55 DEBUG } 19:56:55 DEBUG } 19:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639415" 19:56:55 INFO [http_cmd]: http status code OK 19:56:55 DEBUG --- status code --- 19:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:55 DEBUG --- http response --- 19:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:55 DEBUG ----- output ----- 19:56:55 DEBUG { 19:56:55 DEBUG "status": "success", 19:56:55 DEBUG "data": { 19:56:55 DEBUG "resultType": "vector", 19:56:55 DEBUG "result": [ 19:56:55 DEBUG { 19:56:55 DEBUG "metric": { 19:56:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:55 DEBUG }, 19:56:55 DEBUG "value": [ 19:56:55 DEBUG 19:56:55 DEBUG 19:56:55 DEBUG 19:56:55 DEBUG [...] 19:56:55 DEBUG 19:56:55 DEBUG 19:56:55 DEBUG 19:56:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:55 DEBUG }, 19:56:55 DEBUG "value": [ 19:56:55 DEBUG 1713639415, 19:56:55 DEBUG "2819.4666666666667" 19:56:55 DEBUG ] 19:56:55 DEBUG } 19:56:55 DEBUG ] 19:56:55 DEBUG } 19:56:55 DEBUG } 19:56:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639415" 19:56:55 INFO [http_cmd]: http status code OK 19:56:55 DEBUG --- status code --- 19:56:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:55 DEBUG --- http response --- 19:56:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:56:55 DEBUG ----- output ----- 19:56:55 DEBUG { 19:56:55 DEBUG "status": "success", 19:56:55 DEBUG "data": { 19:56:55 DEBUG "resultType": "vector", 19:56:55 DEBUG "result": [ 19:56:55 DEBUG { 19:56:55 DEBUG "metric": { 19:56:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:56:55 DEBUG }, 19:56:55 DEBUG "value": [ 19:56:55 DEBUG 19:56:55 DEBUG 19:56:55 DEBUG 19:56:55 DEBUG [...] 19:56:55 DEBUG 19:56:55 DEBUG 19:56:55 DEBUG 19:56:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:56:55 DEBUG }, 19:56:55 DEBUG "value": [ 19:56:55 DEBUG 1713639415, 19:56:55 DEBUG "6.829867755189753" 19:56:55 DEBUG ] 19:56:55 DEBUG } 19:56:55 DEBUG ] 19:56:55 DEBUG } 19:56:55 DEBUG } 19:56:55 INFO 19:56:55 INFO [loop_until]: kubectl --namespace=xlou top node 19:56:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:56:55 INFO [loop_until]: OK (rc = 0) 19:56:55 DEBUG --- stdout --- 19:56:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6213Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4618m 15% 7646Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11051m 37% 11371Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3346Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 13177m 44% 13538Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5197Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 8698m 29% 11949Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 6047m 20% 2955Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 11784m 39% 10894Mi 9% gke-xlou-cdm-ds-0f601751-j488 6111m 20% 7184Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 11656m 39% 15365Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 5452m 18% 7859Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 14792m 49% 13077Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 6857m 43% 6350Mi 10% 19:56:55 DEBUG --- stderr --- 19:56:55 DEBUG 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639416" 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 DEBUG --- status code --- 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG { 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG "data": { 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "result": [ 19:56:56 DEBUG { 19:56:56 DEBUG "metric": { 19:56:56 DEBUG "pod": "ds-idrepo-0" 19:56:56 DEBUG }, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG [...] 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG "pod": "ds-idrepo-2" 19:56:56 DEBUG }, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG "0" 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639416" 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 DEBUG --- status code --- 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG { 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG "data": { 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "result": [ 19:56:56 DEBUG { 19:56:56 DEBUG "metric": { 19:56:56 DEBUG "pod": "ds-cts-0" 19:56:56 DEBUG }, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG [...] 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG "pod": "ds-cts-2" 19:56:56 DEBUG }, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG "0" 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639416" 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639416" 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639416" 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639416" 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639416" 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639416" 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639416" 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639416" 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639416" 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639416" 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639416" 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639416" 19:56:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639416" 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 DEBUG --- status code --- 19:56:56 DEBUG --- status code --- 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 DEBUG --- status code --- 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 DEBUG --- status code --- 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 DEBUG --- status code --- 19:56:56 DEBUG --- status code --- 19:56:56 DEBUG --- status code --- 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG --- status code --- 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG --- status code --- 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG --- status code --- 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 DEBUG --- status code --- 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 INFO [http_cmd]: http status code OK 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG --- status code --- 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:56:56 DEBUG --- status code --- 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:56 DEBUG { 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG { 19:56:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG { 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG { 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG --- http response --- 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:56 DEBUG "data": { 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG "data": { 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG { 19:56:56 DEBUG { 19:56:56 DEBUG "data": { 19:56:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 19:56:56 DEBUG { 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG { 19:56:56 DEBUG { 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG { 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "data": { 19:56:56 DEBUG { 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG ----- output ----- 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "result": [ 19:56:56 DEBUG { 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG "result": [ 19:56:56 DEBUG { 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG "data": { 19:56:56 DEBUG "data": { 19:56:56 DEBUG "result": [ 19:56:56 DEBUG { 19:56:56 DEBUG "data": { 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG { 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG "data": { 19:56:56 DEBUG "metric": { 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "data": { 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG { 19:56:56 DEBUG "status": "success", 19:56:56 DEBUG "result": [ 19:56:56 DEBUG "data": { 19:56:56 DEBUG "data": { 19:56:56 DEBUG "metric": { 19:56:56 DEBUG "data": { 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "pod": "ds-idrepo-0" 19:56:56 DEBUG "result": [ 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "result": [ 19:56:56 DEBUG "result": [ 19:56:56 DEBUG "metric": { 19:56:56 DEBUG "data": { 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:56:56 DEBUG "resultType": "vector", 19:56:56 DEBUG "result": [ 19:56:56 DEBUG }, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 19:56:56 DEBUG { 19:56:56 DEBUG { 19:56:56 DEBUG "pod": "ds-cts-0" 19:56:56 DEBUG "result": [ 19:56:56 DEBUG { 19:56:56 DEBUG "result": [ 19:56:56 DEBUG "result": [ 19:56:56 DEBUG }, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG "result": [ 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG { 19:56:56 DEBUG "metric": { 19:56:56 DEBUG "metric": { 19:56:56 DEBUG }, 19:56:56 DEBUG { 19:56:56 DEBUG "metric": { 19:56:56 DEBUG { 19:56:56 DEBUG "metric": { 19:56:56 DEBUG { 19:56:56 DEBUG { 19:56:56 DEBUG { 19:56:56 DEBUG "metric": { 19:56:56 DEBUG "result": [ 19:56:56 DEBUG "metric": { 19:56:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:56 DEBUG }, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG "metric": { 19:56:56 DEBUG "instance": "10.142.0.107:9100" 19:56:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:56 DEBUG 19:56:56 DEBUG "metric": { 19:56:56 DEBUG "metric": { 19:56:56 DEBUG [...] 19:56:56 DEBUG "instance": "10.142.0.107:9100" 19:56:56 DEBUG { 19:56:56 DEBUG "metric": { 19:56:56 DEBUG "pod": "ds-cts-0" 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 19:56:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:56:56 DEBUG }, 19:56:56 DEBUG }, 19:56:56 DEBUG 19:56:56 DEBUG "instance": "10.142.0.107:9100" 19:56:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:56:56 DEBUG }, 19:56:56 DEBUG }, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG "instance": "10.142.0.107:9100" 19:56:56 DEBUG }, 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG }, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 19:56:56 DEBUG }, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:56:56 DEBUG }, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG [...] 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG "value": [ 19:56:56 DEBUG }, 19:56:56 DEBUG 19:56:56 DEBUG "value": [ 19:56:56 DEBUG [...] 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG [...] 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG "pod": "ds-cts-2" 19:56:56 DEBUG }, 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG [...] 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG [...] 19:56:56 DEBUG [...] 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG [...] 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG [...] 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:56:56 DEBUG [...] 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 19:56:56 DEBUG [...] 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG [...] 19:56:56 DEBUG "pod": "ds-cts-2" 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG }, 19:56:56 DEBUG 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG [...] 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG }, 19:56:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:56 DEBUG "instance": "10.142.0.43:9100" 19:56:56 DEBUG 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 19:56:56 DEBUG "0" 19:56:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG 19:56:56 DEBUG "pod": "ds-cts-2" 19:56:56 DEBUG 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG "30" 19:56:56 DEBUG 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG 19:56:56 DEBUG ] 19:56:56 DEBUG }, 19:56:56 DEBUG 19:56:56 DEBUG "instance": "10.142.0.43:9100" 19:56:56 DEBUG 19:56:56 DEBUG }, 19:56:56 DEBUG 19:56:56 DEBUG }, 19:56:56 DEBUG ] 19:56:56 DEBUG }, 19:56:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:56:56 DEBUG "872275968" 19:56:56 DEBUG "instance": "10.142.0.43:9100" 19:56:56 DEBUG } 19:56:56 DEBUG "value": [ 19:56:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:56:56 DEBUG }, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG "value": [ 19:56:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:56:56 DEBUG "value": [ 19:56:56 DEBUG } 19:56:56 DEBUG "value": [ 19:56:56 DEBUG }, 19:56:56 DEBUG ] 19:56:56 DEBUG }, 19:56:56 DEBUG ] 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG }, 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG }, 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG ] 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG "value": [ 19:56:56 DEBUG } 19:56:56 DEBUG "value": [ 19:56:56 DEBUG } 19:56:56 DEBUG "0" 19:56:56 DEBUG "value": [ 19:56:56 DEBUG "0.16346666666660498" 19:56:56 DEBUG ] 19:56:56 DEBUG "instance": "10.142.0.43:9100" 19:56:56 DEBUG "value": [ 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG } 19:56:56 DEBUG "80316040.46666667" 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG ] 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG } 19:56:56 DEBUG ] 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG } 19:56:56 DEBUG "0.000036431111111111106" 19:56:56 DEBUG }, 19:56:56 DEBUG "0" 19:56:56 DEBUG "358345.5" 19:56:56 DEBUG } 19:56:56 DEBUG ] 19:56:56 DEBUG "0.1565" 19:56:56 DEBUG } 19:56:56 DEBUG "80311007.26666667" 19:56:56 DEBUG } 19:56:56 DEBUG "2383643.458672674" 19:56:56 DEBUG ] 19:56:56 DEBUG ] 19:56:56 DEBUG "value": [ 19:56:56 DEBUG ] 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG ] 19:56:56 DEBUG ] 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG 1713639416, 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG ] 19:56:56 DEBUG "0.06633333333332037" 19:56:56 DEBUG ] 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG ] 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG } 19:56:56 DEBUG ] 19:56:56 DEBUG } 19:56:56 DEBUG } 19:57:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639472" 19:57:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639472" 19:57:52 ERROR [http_cmd]: ERROR 19:57:52 INFO [http_cmd]: http status code OK 19:57:52 ERROR --- status code --- 19:57:52 DEBUG --- status code --- 19:57:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:57:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:52 ERROR --- http response --- 19:57:52 DEBUG --- http response --- 19:57:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:57:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:57:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:57:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639473" 19:57:53 INFO [http_cmd]: http status code OK 19:57:53 DEBUG --- status code --- 19:57:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:53 DEBUG --- http response --- 19:57:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:53 DEBUG ----- output ----- 19:57:53 DEBUG { 19:57:53 DEBUG "status": "success", 19:57:53 DEBUG "data": { 19:57:53 DEBUG "resultType": "vector", 19:57:53 DEBUG "result": [ 19:57:53 DEBUG { 19:57:53 DEBUG "metric": { 19:57:53 DEBUG "pod": "ds-idrepo-0" 19:57:53 DEBUG }, 19:57:53 DEBUG "value": [ 19:57:53 DEBUG 19:57:53 DEBUG 19:57:53 DEBUG 19:57:53 DEBUG [...] 19:57:53 DEBUG 19:57:53 DEBUG 19:57:53 DEBUG 19:57:53 DEBUG "pod": "ds-idrepo-2" 19:57:53 DEBUG }, 19:57:53 DEBUG "value": [ 19:57:53 DEBUG 1713639473, 19:57:53 DEBUG "0" 19:57:53 DEBUG ] 19:57:53 DEBUG } 19:57:53 DEBUG ] 19:57:53 DEBUG } 19:57:53 DEBUG } 19:57:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639473" 19:57:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639473" 19:57:53 INFO [http_cmd]: http status code OK 19:57:53 DEBUG --- status code --- 19:57:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:53 DEBUG --- http response --- 19:57:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:53 DEBUG ----- output ----- 19:57:53 DEBUG { 19:57:53 DEBUG "status": "success", 19:57:53 DEBUG "data": { 19:57:53 INFO [http_cmd]: http status code OK 19:57:53 DEBUG "resultType": "vector", 19:57:53 DEBUG --- status code --- 19:57:53 DEBUG "result": [ 19:57:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:53 DEBUG { 19:57:53 DEBUG --- http response --- 19:57:53 DEBUG "metric": { 19:57:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:53 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:53 DEBUG ----- output ----- 19:57:53 DEBUG }, 19:57:53 DEBUG { 19:57:53 DEBUG "value": [ 19:57:53 DEBUG "status": "success", 19:57:53 DEBUG "data": { 19:57:53 DEBUG 19:57:53 DEBUG "resultType": "vector", 19:57:53 DEBUG 19:57:53 DEBUG "result": [ 19:57:53 DEBUG 19:57:53 DEBUG { 19:57:53 DEBUG [...] 19:57:53 DEBUG "metric": { 19:57:53 DEBUG 19:57:53 DEBUG "pod": "ds-idrepo-0" 19:57:53 DEBUG 19:57:53 DEBUG }, 19:57:53 DEBUG 19:57:53 DEBUG "value": [ 19:57:53 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:53 DEBUG 19:57:53 DEBUG }, 19:57:53 DEBUG 19:57:53 DEBUG "value": [ 19:57:53 DEBUG 19:57:53 DEBUG 1713639473, 19:57:53 DEBUG [...] 19:57:53 DEBUG "2669.5" 19:57:53 DEBUG 19:57:53 DEBUG ] 19:57:53 DEBUG } 19:57:53 DEBUG ] 19:57:53 DEBUG } 19:57:53 DEBUG 19:57:53 DEBUG } 19:57:53 DEBUG 19:57:53 DEBUG "pod": "ds-idrepo-2" 19:57:53 DEBUG }, 19:57:53 DEBUG "value": [ 19:57:53 DEBUG 1713639473, 19:57:53 DEBUG "0" 19:57:53 DEBUG ] 19:57:53 DEBUG } 19:57:53 DEBUG ] 19:57:53 DEBUG } 19:57:53 DEBUG } 19:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639474" 19:57:54 INFO 19:57:54 INFO [loop_until]: kubectl --namespace=xlou top pods 19:57:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:57:54 INFO [http_cmd]: http status code OK 19:57:54 DEBUG --- status code --- 19:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:54 DEBUG --- http response --- 19:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:57:54 DEBUG ----- output ----- 19:57:54 DEBUG { 19:57:54 DEBUG "status": "success", 19:57:54 DEBUG "data": { 19:57:54 DEBUG "resultType": "vector", 19:57:54 DEBUG "result": [ 19:57:54 DEBUG { 19:57:54 DEBUG "metric": { 19:57:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:57:54 DEBUG }, 19:57:54 DEBUG "value": [ 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG [...] 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:57:54 DEBUG }, 19:57:54 DEBUG "value": [ 19:57:54 DEBUG 1713639474, 19:57:54 DEBUG "0" 19:57:54 DEBUG ] 19:57:54 DEBUG } 19:57:54 DEBUG ] 19:57:54 DEBUG } 19:57:54 DEBUG } 19:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639474" 19:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639474" 19:57:54 INFO [http_cmd]: http status code OK 19:57:54 DEBUG --- status code --- 19:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:54 DEBUG --- http response --- 19:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:54 DEBUG ----- output ----- 19:57:54 DEBUG { 19:57:54 DEBUG "status": "success", 19:57:54 DEBUG "data": { 19:57:54 DEBUG "resultType": "vector", 19:57:54 DEBUG "result": [ 19:57:54 DEBUG { 19:57:54 INFO [http_cmd]: http status code OK 19:57:54 DEBUG --- status code --- 19:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:54 DEBUG --- http response --- 19:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:54 DEBUG "metric": { 19:57:54 DEBUG ----- output ----- 19:57:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:57:54 DEBUG { 19:57:54 DEBUG }, 19:57:54 DEBUG "status": "success", 19:57:54 DEBUG "value": [ 19:57:54 DEBUG "data": { 19:57:54 DEBUG 19:57:54 DEBUG "resultType": "vector", 19:57:54 DEBUG "result": [ 19:57:54 DEBUG { 19:57:54 DEBUG "metric": { 19:57:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:54 DEBUG }, 19:57:54 DEBUG "value": [ 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG [...] 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:54 DEBUG }, 19:57:54 DEBUG "value": [ 19:57:54 DEBUG 1713639474, 19:57:54 DEBUG "2837.9" 19:57:54 DEBUG ] 19:57:54 DEBUG } 19:57:54 DEBUG ] 19:57:54 DEBUG } 19:57:54 DEBUG } 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG [...] 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:57:54 DEBUG }, 19:57:54 DEBUG "value": [ 19:57:54 DEBUG 1713639474, 19:57:54 DEBUG "9.573525348983864" 19:57:54 DEBUG ] 19:57:54 DEBUG } 19:57:54 DEBUG ] 19:57:54 DEBUG } 19:57:54 DEBUG } 19:57:54 INFO [loop_until]: OK (rc = 0) 19:57:54 DEBUG --- stdout --- 19:57:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17973m 8650Mi am-767f6b67b5-qcqz2 18372m 9065Mi am-767f6b67b5-vbltz 19743m 8667Mi ds-cts-0 18393m 10373Mi ds-cts-1 17053m 10326Mi ds-cts-2 19750m 10349Mi ds-idrepo-0 7778m 3894Mi ds-idrepo-1 7826m 3883Mi ds-idrepo-2 7808m 3771Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 14m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9597m 746Mi 19:57:54 DEBUG --- stderr --- 19:57:54 DEBUG 19:57:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639474" 19:57:54 INFO [http_cmd]: http status code OK 19:57:54 DEBUG --- status code --- 19:57:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:54 DEBUG --- http response --- 19:57:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:54 DEBUG ----- output ----- 19:57:54 DEBUG { 19:57:54 DEBUG "status": "success", 19:57:54 DEBUG "data": { 19:57:54 DEBUG "resultType": "vector", 19:57:54 DEBUG "result": [ 19:57:54 DEBUG { 19:57:54 DEBUG "metric": { 19:57:54 DEBUG "instance": "10.142.0.107:9100" 19:57:54 DEBUG }, 19:57:54 DEBUG "value": [ 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG [...] 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG 19:57:54 DEBUG "instance": "10.142.0.43:9100" 19:57:54 DEBUG }, 19:57:54 DEBUG "value": [ 19:57:54 DEBUG 1713639474, 19:57:54 DEBUG "3635797718016" 19:57:54 DEBUG ] 19:57:54 DEBUG } 19:57:54 DEBUG ] 19:57:54 DEBUG } 19:57:54 DEBUG } 19:57:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639475" 19:57:55 INFO [http_cmd]: http status code OK 19:57:55 DEBUG --- status code --- 19:57:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:55 DEBUG --- http response --- 19:57:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:55 DEBUG ----- output ----- 19:57:55 DEBUG { 19:57:55 DEBUG "status": "success", 19:57:55 DEBUG "data": { 19:57:55 DEBUG "resultType": "vector", 19:57:55 DEBUG "result": [ 19:57:55 DEBUG { 19:57:55 DEBUG "metric": { 19:57:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:55 DEBUG }, 19:57:55 DEBUG "value": [ 19:57:55 DEBUG 19:57:55 DEBUG 19:57:55 DEBUG 19:57:55 DEBUG [...] 19:57:55 DEBUG 19:57:55 DEBUG 19:57:55 DEBUG 19:57:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:55 DEBUG }, 19:57:55 DEBUG "value": [ 19:57:55 DEBUG 1713639475, 19:57:55 DEBUG "2838.7666666666664" 19:57:55 DEBUG ] 19:57:55 DEBUG } 19:57:55 DEBUG ] 19:57:55 DEBUG } 19:57:55 DEBUG } 19:57:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639475" 19:57:55 INFO [http_cmd]: http status code OK 19:57:55 DEBUG --- status code --- 19:57:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:55 DEBUG --- http response --- 19:57:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:55 DEBUG ----- output ----- 19:57:55 DEBUG { 19:57:55 DEBUG "status": "success", 19:57:55 DEBUG "data": { 19:57:55 DEBUG "resultType": "vector", 19:57:55 DEBUG "result": [ 19:57:55 DEBUG { 19:57:55 DEBUG "metric": { 19:57:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:55 DEBUG }, 19:57:55 DEBUG "value": [ 19:57:55 DEBUG 19:57:55 DEBUG 19:57:55 DEBUG 19:57:55 DEBUG [...] 19:57:55 DEBUG 19:57:55 DEBUG 19:57:55 DEBUG 19:57:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:55 DEBUG }, 19:57:55 DEBUG "value": [ 19:57:55 DEBUG 1713639475, 19:57:55 DEBUG "2838.3" 19:57:55 DEBUG ] 19:57:55 DEBUG } 19:57:55 DEBUG ] 19:57:55 DEBUG } 19:57:55 DEBUG } 19:57:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639475" 19:57:55 INFO [http_cmd]: http status code OK 19:57:55 DEBUG --- status code --- 19:57:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:55 DEBUG --- http response --- 19:57:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:55 DEBUG ----- output ----- 19:57:55 DEBUG { 19:57:55 DEBUG "status": "success", 19:57:55 DEBUG "data": { 19:57:55 DEBUG "resultType": "vector", 19:57:55 DEBUG "result": [ 19:57:55 DEBUG { 19:57:55 DEBUG "metric": { 19:57:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:57:55 DEBUG }, 19:57:55 DEBUG "value": [ 19:57:55 DEBUG 19:57:55 DEBUG 19:57:55 DEBUG 19:57:55 DEBUG [...] 19:57:55 DEBUG 19:57:55 DEBUG 19:57:55 DEBUG 19:57:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:57:55 DEBUG }, 19:57:55 DEBUG "value": [ 19:57:55 DEBUG 1713639475, 19:57:55 DEBUG "16.310895197875173" 19:57:55 DEBUG ] 19:57:55 DEBUG } 19:57:55 DEBUG ] 19:57:55 DEBUG } 19:57:55 DEBUG } 19:57:55 INFO 19:57:55 INFO [loop_until]: kubectl --namespace=xlou top node 19:57:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:57:56 INFO [loop_until]: OK (rc = 0) 19:57:56 DEBUG --- stdout --- 19:57:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6224Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8129m 27% 7658Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17913m 60% 11377Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3336Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18730m 62% 13561Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5198Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18259m 61% 11958Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15627m 52% 2968Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20534m 68% 10893Mi 9% gke-xlou-cdm-ds-0f601751-j488 8381m 28% 7169Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17126m 57% 15362Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8642m 28% 7859Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19595m 65% 13017Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9805m 61% 6387Mi 10% 19:57:56 DEBUG --- stderr --- 19:57:56 DEBUG 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639476" 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG { 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG "data": { 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG "result": [ 19:57:56 DEBUG { 19:57:56 DEBUG "metric": { 19:57:56 DEBUG "pod": "ds-idrepo-0" 19:57:56 DEBUG }, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG [...] 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "pod": "ds-idrepo-2" 19:57:56 DEBUG }, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG "0" 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639476" 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG { 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG "data": { 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG "result": [ 19:57:56 DEBUG { 19:57:56 DEBUG "metric": { 19:57:56 DEBUG "pod": "ds-cts-0" 19:57:56 DEBUG }, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG [...] 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "pod": "ds-cts-2" 19:57:56 DEBUG }, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG "0" 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639476" 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG { 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG "data": { 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG "result": [ 19:57:56 DEBUG { 19:57:56 DEBUG "metric": { 19:57:56 DEBUG "pod": "ds-idrepo-0" 19:57:56 DEBUG }, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG [...] 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "pod": "ds-cts-2" 19:57:56 DEBUG }, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG "0" 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639476" 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639476" 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639476" 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639476" 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639476" 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639476" 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639476" 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639476" 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639476" 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639476" 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639476" 19:57:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639476" 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 DEBUG --- status code --- 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 DEBUG --- status code --- 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG --- http response --- 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG --- http response --- 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:56 INFO [http_cmd]: http status code OK 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG --- status code --- 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG { 19:57:56 DEBUG { 19:57:56 DEBUG { 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG "data": { 19:57:56 DEBUG { 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG { 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG { 19:57:56 DEBUG { 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG "data": { 19:57:56 DEBUG { 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG "data": { 19:57:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:57:56 DEBUG "data": { 19:57:56 DEBUG "data": { 19:57:56 DEBUG { 19:57:56 DEBUG "result": [ 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG "data": { 19:57:56 DEBUG "data": { 19:57:56 DEBUG { 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG --- http response --- 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG { 19:57:56 DEBUG "result": [ 19:57:56 DEBUG "data": { 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG "result": [ 19:57:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:57:56 DEBUG "result": [ 19:57:56 DEBUG { 19:57:56 DEBUG "data": { 19:57:56 DEBUG "metric": { 19:57:56 DEBUG "instance": "10.142.0.107:9100" 19:57:56 DEBUG { 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG "result": [ 19:57:56 DEBUG "result": [ 19:57:56 DEBUG "data": { 19:57:56 DEBUG { 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG "result": [ 19:57:56 DEBUG "metric": { 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG }, 19:57:56 DEBUG ----- output ----- 19:57:56 DEBUG "metric": { 19:57:56 DEBUG "result": [ 19:57:56 DEBUG { 19:57:56 DEBUG { 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG "metric": { 19:57:56 DEBUG { 19:57:56 DEBUG { 19:57:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:57:56 DEBUG "result": [ 19:57:56 DEBUG "value": [ 19:57:56 DEBUG { 19:57:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:56 DEBUG }, 19:57:56 DEBUG "metric": { 19:57:56 DEBUG "metric": { 19:57:56 DEBUG "result": [ 19:57:56 DEBUG "pod": "ds-cts-0" 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG "metric": { 19:57:56 DEBUG "pod": "ds-cts-0" 19:57:56 DEBUG { 19:57:56 DEBUG 19:57:56 DEBUG "status": "success", 19:57:56 DEBUG "data": { 19:57:56 DEBUG "value": [ 19:57:56 DEBUG "instance": "10.142.0.107:9100" 19:57:56 DEBUG }, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG [...] 19:57:56 DEBUG 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG { 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG { 19:57:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:57:56 DEBUG }, 19:57:56 DEBUG "metric": { 19:57:56 DEBUG }, 19:57:56 DEBUG 19:57:56 DEBUG }, 19:57:56 DEBUG "data": { 19:57:56 DEBUG "result": [ 19:57:56 DEBUG "metric": { 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "metric": { 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 19:57:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:56 DEBUG }, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG "value": [ 19:57:56 DEBUG "resultType": "vector", 19:57:56 DEBUG { 19:57:56 DEBUG "instance": "10.142.0.107:9100" 19:57:56 DEBUG [...] 19:57:56 DEBUG 19:57:56 DEBUG [...] 19:57:56 DEBUG }, 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "result": [ 19:57:56 DEBUG "metric": { 19:57:56 DEBUG }, 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 19:57:56 DEBUG "pod": "am-767f6b67b5-9zw22" 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG [...] 19:57:56 DEBUG 19:57:56 DEBUG { 19:57:56 DEBUG "instance": "10.142.0.107:9100" 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG [...] 19:57:56 DEBUG }, 19:57:56 DEBUG "instance": "10.142.0.43:9100" 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "value": [ 19:57:56 DEBUG }, 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "instance": "10.142.0.43:9100" 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG }, 19:57:56 DEBUG 19:57:56 DEBUG [...] 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG }, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 19:57:56 DEBUG "value": [ 19:57:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:56 DEBUG "metric": { 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG }, 19:57:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:57:56 DEBUG 19:57:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:57:56 DEBUG }, 19:57:56 DEBUG 19:57:56 DEBUG [...] 19:57:56 DEBUG "85004431.3" 19:57:56 DEBUG ] 19:57:56 DEBUG 19:57:56 DEBUG "0.1714666666667" 19:57:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:57:56 DEBUG "value": [ 19:57:56 DEBUG }, 19:57:56 DEBUG }, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG "value": [ 19:57:56 DEBUG [...] 19:57:56 DEBUG 19:57:56 DEBUG } 19:57:56 DEBUG [...] 19:57:56 DEBUG ] 19:57:56 DEBUG }, 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG } 19:57:56 DEBUG 19:57:56 DEBUG "value": [ 19:57:56 DEBUG "368212" 19:57:56 DEBUG 19:57:56 DEBUG [...] 19:57:56 DEBUG "879140864" 19:57:56 DEBUG "0" 19:57:56 DEBUG ] 19:57:56 DEBUG "instance": "10.142.0.43:9100" 19:57:56 DEBUG }, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG ] 19:57:56 DEBUG [...] 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG 19:57:56 DEBUG ] 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG "0.06866666666665917" 19:57:56 DEBUG } 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "1566599.3738147658" 19:57:56 DEBUG ] 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG ] 19:57:56 DEBUG 19:57:56 DEBUG } 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "pod": "ds-cts-2" 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG 19:57:56 DEBUG 19:57:56 DEBUG "instance": "10.142.0.43:9100" 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG [...] 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG "pod": "am-767f6b67b5-vbltz" 19:57:56 DEBUG "pod": "ds-cts-2" 19:57:56 DEBUG }, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 19:57:56 DEBUG } 19:57:56 DEBUG }, 19:57:56 DEBUG ] 19:57:56 DEBUG }, 19:57:56 DEBUG }, 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG ] 19:57:56 DEBUG 19:57:56 DEBUG "value": [ 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG "value": [ 19:57:56 DEBUG "85009295.86666667" 19:57:56 DEBUG } 19:57:56 DEBUG 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG "value": [ 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 19:57:56 DEBUG }, 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG "0.157" 19:57:56 DEBUG } 19:57:56 DEBUG "26.566666666666666" 19:57:56 DEBUG "value": [ 19:57:56 DEBUG "0.00009999999999999992" 19:57:56 DEBUG ] 19:57:56 DEBUG ] 19:57:56 DEBUG ] 19:57:56 DEBUG 1713639476, 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG "0" 19:57:56 DEBUG } 19:57:56 DEBUG ] 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG } 19:57:56 DEBUG ] 19:57:56 DEBUG } 19:57:56 DEBUG } 19:58:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639532" 19:58:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639532" 19:58:52 ERROR [http_cmd]: ERROR 19:58:52 ERROR --- status code --- 19:58:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:58:52 ERROR --- http response --- 19:58:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:58:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:58:52 INFO [http_cmd]: http status code OK 19:58:52 DEBUG --- status code --- 19:58:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:52 DEBUG --- http response --- 19:58:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:58:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639533" 19:58:53 INFO [http_cmd]: http status code OK 19:58:53 DEBUG --- status code --- 19:58:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:53 DEBUG --- http response --- 19:58:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:53 DEBUG ----- output ----- 19:58:53 DEBUG { 19:58:53 DEBUG "status": "success", 19:58:53 DEBUG "data": { 19:58:53 DEBUG "resultType": "vector", 19:58:53 DEBUG "result": [ 19:58:53 DEBUG { 19:58:53 DEBUG "metric": { 19:58:53 DEBUG "pod": "ds-idrepo-0" 19:58:53 DEBUG }, 19:58:53 DEBUG "value": [ 19:58:53 DEBUG 19:58:53 DEBUG 19:58:53 DEBUG 19:58:53 DEBUG [...] 19:58:53 DEBUG 19:58:53 DEBUG 19:58:53 DEBUG 19:58:53 DEBUG "pod": "ds-idrepo-2" 19:58:53 DEBUG }, 19:58:53 DEBUG "value": [ 19:58:53 DEBUG 1713639533, 19:58:53 DEBUG "0" 19:58:53 DEBUG ] 19:58:53 DEBUG } 19:58:53 DEBUG ] 19:58:53 DEBUG } 19:58:53 DEBUG } 19:58:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639533" 19:58:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639533" 19:58:54 INFO [http_cmd]: http status code OK 19:58:54 DEBUG --- status code --- 19:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:54 DEBUG --- http response --- 19:58:54 INFO [http_cmd]: http status code OK 19:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:54 DEBUG --- status code --- 19:58:54 DEBUG ----- output ----- 19:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:54 DEBUG { 19:58:54 DEBUG --- http response --- 19:58:54 DEBUG "status": "success", 19:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:54 DEBUG "data": { 19:58:54 DEBUG ----- output ----- 19:58:54 DEBUG "resultType": "vector", 19:58:54 DEBUG { 19:58:54 DEBUG "result": [ 19:58:54 DEBUG "status": "success", 19:58:54 DEBUG { 19:58:54 DEBUG "data": { 19:58:54 DEBUG "metric": { 19:58:54 DEBUG "resultType": "vector", 19:58:54 DEBUG "pod": "ds-idrepo-0" 19:58:54 DEBUG "result": [ 19:58:54 DEBUG }, 19:58:54 DEBUG { 19:58:54 DEBUG "value": [ 19:58:54 DEBUG "metric": { 19:58:54 DEBUG 19:58:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:54 DEBUG 19:58:54 DEBUG }, 19:58:54 DEBUG 19:58:54 DEBUG "value": [ 19:58:54 DEBUG [...] 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG [...] 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG "pod": "ds-idrepo-2" 19:58:54 DEBUG 19:58:54 DEBUG }, 19:58:54 DEBUG 19:58:54 DEBUG "value": [ 19:58:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:54 DEBUG 1713639533, 19:58:54 DEBUG }, 19:58:54 DEBUG "0" 19:58:54 DEBUG "value": [ 19:58:54 DEBUG ] 19:58:54 DEBUG 1713639533, 19:58:54 DEBUG } 19:58:54 DEBUG "4.866666666666666" 19:58:54 DEBUG ] 19:58:54 DEBUG ] 19:58:54 DEBUG } 19:58:54 DEBUG } 19:58:54 DEBUG } 19:58:54 DEBUG ] 19:58:54 DEBUG } 19:58:54 DEBUG } 19:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639534" 19:58:54 INFO [http_cmd]: http status code OK 19:58:54 DEBUG --- status code --- 19:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:54 DEBUG --- http response --- 19:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:58:54 DEBUG ----- output ----- 19:58:54 DEBUG { 19:58:54 DEBUG "status": "success", 19:58:54 DEBUG "data": { 19:58:54 DEBUG "resultType": "vector", 19:58:54 DEBUG "result": [ 19:58:54 DEBUG { 19:58:54 DEBUG "metric": { 19:58:54 DEBUG "pod": "ds-cts-1" 19:58:54 DEBUG }, 19:58:54 DEBUG "value": [ 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG [...] 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG "pod": "ds-cts-0" 19:58:54 DEBUG }, 19:58:54 DEBUG "value": [ 19:58:54 DEBUG 1713639534, 19:58:54 DEBUG "57532907.97198841" 19:58:54 DEBUG ] 19:58:54 DEBUG } 19:58:54 DEBUG ] 19:58:54 DEBUG } 19:58:54 DEBUG } 19:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639534" 19:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639534" 19:58:54 INFO [http_cmd]: http status code OK 19:58:54 DEBUG --- status code --- 19:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:54 DEBUG --- http response --- 19:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:54 DEBUG ----- output ----- 19:58:54 DEBUG { 19:58:54 DEBUG "status": "success", 19:58:54 DEBUG "data": { 19:58:54 DEBUG "resultType": "vector", 19:58:54 DEBUG "result": [ 19:58:54 INFO [http_cmd]: http status code OK 19:58:54 DEBUG { 19:58:54 DEBUG --- status code --- 19:58:54 DEBUG "metric": { 19:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:54 DEBUG --- http response --- 19:58:54 DEBUG }, 19:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:54 DEBUG "value": [ 19:58:54 DEBUG ----- output ----- 19:58:54 DEBUG 19:58:54 DEBUG { 19:58:54 DEBUG 19:58:54 DEBUG "status": "success", 19:58:54 DEBUG 19:58:54 DEBUG "data": { 19:58:54 DEBUG [...] 19:58:54 DEBUG "resultType": "vector", 19:58:54 DEBUG 19:58:54 DEBUG "result": [ 19:58:54 DEBUG 19:58:54 DEBUG { 19:58:54 DEBUG 19:58:54 DEBUG "metric": { 19:58:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:58:54 DEBUG }, 19:58:54 DEBUG }, 19:58:54 DEBUG "value": [ 19:58:54 DEBUG "value": [ 19:58:54 DEBUG 1713639534, 19:58:54 DEBUG 19:58:54 DEBUG "2841.0333333333333" 19:58:54 DEBUG 19:58:54 DEBUG ] 19:58:54 DEBUG 19:58:54 DEBUG } 19:58:54 DEBUG [...] 19:58:54 DEBUG ] 19:58:54 DEBUG 19:58:54 DEBUG } 19:58:54 DEBUG 19:58:54 DEBUG } 19:58:54 DEBUG 19:58:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:58:54 DEBUG }, 19:58:54 DEBUG "value": [ 19:58:54 DEBUG 1713639534, 19:58:54 DEBUG "9.628120412745176" 19:58:54 DEBUG ] 19:58:54 DEBUG } 19:58:54 DEBUG ] 19:58:54 DEBUG } 19:58:54 DEBUG } 19:58:54 INFO 19:58:54 INFO [loop_until]: kubectl --namespace=xlou top pods 19:58:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:58:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639534" 19:58:54 INFO [http_cmd]: http status code OK 19:58:54 DEBUG --- status code --- 19:58:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:54 DEBUG --- http response --- 19:58:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:54 DEBUG ----- output ----- 19:58:54 DEBUG { 19:58:54 DEBUG "status": "success", 19:58:54 DEBUG "data": { 19:58:54 DEBUG "resultType": "vector", 19:58:54 DEBUG "result": [ 19:58:54 DEBUG { 19:58:54 DEBUG "metric": { 19:58:54 DEBUG "instance": "10.142.0.107:9100" 19:58:54 DEBUG }, 19:58:54 DEBUG "value": [ 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG [...] 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG 19:58:54 DEBUG "instance": "10.142.0.43:9100" 19:58:54 DEBUG }, 19:58:54 DEBUG "value": [ 19:58:54 DEBUG 1713639534, 19:58:54 DEBUG "3639508657152" 19:58:54 DEBUG ] 19:58:54 DEBUG } 19:58:54 DEBUG ] 19:58:54 DEBUG } 19:58:54 DEBUG } 19:58:54 INFO [loop_until]: OK (rc = 0) 19:58:54 DEBUG --- stdout --- 19:58:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17533m 8650Mi am-767f6b67b5-qcqz2 16831m 9065Mi am-767f6b67b5-vbltz 19627m 8667Mi ds-cts-0 18996m 10353Mi ds-cts-1 16297m 10327Mi ds-cts-2 19925m 10355Mi ds-idrepo-0 7696m 3893Mi ds-idrepo-1 7687m 3882Mi ds-idrepo-2 7637m 3771Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 16m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9415m 786Mi 19:58:54 DEBUG --- stderr --- 19:58:54 DEBUG 19:58:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639535" 19:58:55 INFO [http_cmd]: http status code OK 19:58:55 DEBUG --- status code --- 19:58:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:55 DEBUG --- http response --- 19:58:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:55 DEBUG ----- output ----- 19:58:55 DEBUG { 19:58:55 DEBUG "status": "success", 19:58:55 DEBUG "data": { 19:58:55 DEBUG "resultType": "vector", 19:58:55 DEBUG "result": [ 19:58:55 DEBUG { 19:58:55 DEBUG "metric": { 19:58:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:55 DEBUG }, 19:58:55 DEBUG "value": [ 19:58:55 DEBUG 19:58:55 DEBUG 19:58:55 DEBUG 19:58:55 DEBUG [...] 19:58:55 DEBUG 19:58:55 DEBUG 19:58:55 DEBUG 19:58:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:55 DEBUG }, 19:58:55 DEBUG "value": [ 19:58:55 DEBUG 1713639535, 19:58:55 DEBUG "2841.0666666666666" 19:58:55 DEBUG ] 19:58:55 DEBUG } 19:58:55 DEBUG ] 19:58:55 DEBUG } 19:58:55 DEBUG } 19:58:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639535" 19:58:55 INFO [http_cmd]: http status code OK 19:58:55 DEBUG --- status code --- 19:58:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:55 DEBUG --- http response --- 19:58:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:55 DEBUG ----- output ----- 19:58:55 DEBUG { 19:58:55 DEBUG "status": "success", 19:58:55 DEBUG "data": { 19:58:55 DEBUG "resultType": "vector", 19:58:55 DEBUG "result": [ 19:58:55 DEBUG { 19:58:55 DEBUG "metric": { 19:58:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:55 DEBUG }, 19:58:55 DEBUG "value": [ 19:58:55 DEBUG 19:58:55 DEBUG 19:58:55 DEBUG 19:58:55 DEBUG [...] 19:58:55 DEBUG 19:58:55 DEBUG 19:58:55 DEBUG 19:58:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:55 DEBUG }, 19:58:55 DEBUG "value": [ 19:58:55 DEBUG 1713639535, 19:58:55 DEBUG "2840.866666666667" 19:58:55 DEBUG ] 19:58:55 DEBUG } 19:58:55 DEBUG ] 19:58:55 DEBUG } 19:58:55 DEBUG } 19:58:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639535" 19:58:55 INFO [http_cmd]: http status code OK 19:58:55 DEBUG --- status code --- 19:58:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:55 DEBUG --- http response --- 19:58:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:58:55 DEBUG ----- output ----- 19:58:55 DEBUG { 19:58:55 DEBUG "status": "success", 19:58:55 DEBUG "data": { 19:58:55 DEBUG "resultType": "vector", 19:58:55 DEBUG "result": [ 19:58:55 DEBUG { 19:58:55 DEBUG "metric": { 19:58:55 DEBUG "pod": "ds-idrepo-0" 19:58:55 DEBUG }, 19:58:55 DEBUG "value": [ 19:58:55 DEBUG 19:58:55 DEBUG 19:58:55 DEBUG 19:58:55 DEBUG [...] 19:58:55 DEBUG 19:58:55 DEBUG 19:58:55 DEBUG 19:58:55 DEBUG "pod": "ds-cts-0" 19:58:55 DEBUG }, 19:58:55 DEBUG "value": [ 19:58:55 DEBUG 1713639535, 19:58:55 DEBUG "16.49961749310395" 19:58:55 DEBUG ] 19:58:55 DEBUG } 19:58:55 DEBUG ] 19:58:55 DEBUG } 19:58:55 DEBUG } 19:58:56 INFO 19:58:56 INFO [loop_until]: kubectl --namespace=xlou top node 19:58:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:58:56 INFO [loop_until]: OK (rc = 0) 19:58:56 DEBUG --- stdout --- 19:58:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6224Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7963m 26% 7634Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17315m 57% 11366Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3328Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 17787m 59% 13530Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5205Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18099m 60% 11959Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15663m 52% 2968Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19476m 65% 10890Mi 9% gke-xlou-cdm-ds-0f601751-j488 8659m 29% 7168Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16418m 54% 15370Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8405m 28% 7855Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 17903m 59% 13011Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9833m 61% 6389Mi 10% 19:58:56 DEBUG --- stderr --- 19:58:56 DEBUG 19:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639536" 19:58:56 INFO [http_cmd]: http status code OK 19:58:56 DEBUG --- status code --- 19:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:56 DEBUG --- http response --- 19:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:56 DEBUG ----- output ----- 19:58:56 DEBUG { 19:58:56 DEBUG "status": "success", 19:58:56 DEBUG "data": { 19:58:56 DEBUG "resultType": "vector", 19:58:56 DEBUG "result": [ 19:58:56 DEBUG { 19:58:56 DEBUG "metric": { 19:58:56 DEBUG "pod": "ds-idrepo-0" 19:58:56 DEBUG }, 19:58:56 DEBUG "value": [ 19:58:56 DEBUG 19:58:56 DEBUG 19:58:56 DEBUG 19:58:56 DEBUG [...] 19:58:56 DEBUG 19:58:56 DEBUG 19:58:56 DEBUG 19:58:56 DEBUG "pod": "ds-idrepo-2" 19:58:56 DEBUG }, 19:58:56 DEBUG "value": [ 19:58:56 DEBUG 1713639536, 19:58:56 DEBUG "0" 19:58:56 DEBUG ] 19:58:56 DEBUG } 19:58:56 DEBUG ] 19:58:56 DEBUG } 19:58:56 DEBUG } 19:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639536" 19:58:56 INFO [http_cmd]: http status code OK 19:58:56 DEBUG --- status code --- 19:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:56 DEBUG --- http response --- 19:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:58:56 DEBUG ----- output ----- 19:58:56 DEBUG { 19:58:56 DEBUG "status": "success", 19:58:56 DEBUG "data": { 19:58:56 DEBUG "resultType": "vector", 19:58:56 DEBUG "result": [ 19:58:56 DEBUG { 19:58:56 DEBUG "metric": { 19:58:56 DEBUG "pod": "ds-cts-0" 19:58:56 DEBUG }, 19:58:56 DEBUG "value": [ 19:58:56 DEBUG 19:58:56 DEBUG 19:58:56 DEBUG 19:58:56 DEBUG [...] 19:58:56 DEBUG 19:58:56 DEBUG 19:58:56 DEBUG 19:58:56 DEBUG "pod": "ds-cts-2" 19:58:56 DEBUG }, 19:58:56 DEBUG "value": [ 19:58:56 DEBUG 1713639536, 19:58:56 DEBUG "0" 19:58:56 DEBUG ] 19:58:56 DEBUG } 19:58:56 DEBUG ] 19:58:56 DEBUG } 19:58:56 DEBUG } 19:58:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639536" 19:58:56 INFO [http_cmd]: http status code OK 19:58:56 DEBUG --- status code --- 19:58:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:56 DEBUG --- http response --- 19:58:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:58:56 DEBUG ----- output ----- 19:58:56 DEBUG { 19:58:56 DEBUG "status": "success", 19:58:56 DEBUG "data": { 19:58:56 DEBUG "resultType": "vector", 19:58:56 DEBUG "result": [ 19:58:56 DEBUG { 19:58:56 DEBUG "metric": { 19:58:56 DEBUG "pod": "ds-idrepo-0" 19:58:56 DEBUG }, 19:58:56 DEBUG "value": [ 19:58:56 DEBUG 19:58:56 DEBUG 19:58:56 DEBUG 19:58:56 DEBUG [...] 19:58:56 DEBUG 19:58:56 DEBUG 19:58:56 DEBUG 19:58:56 DEBUG "pod": "ds-cts-2" 19:58:56 DEBUG }, 19:58:56 DEBUG "value": [ 19:58:56 DEBUG 1713639536, 19:58:56 DEBUG "0" 19:58:56 DEBUG ] 19:58:56 DEBUG } 19:58:56 DEBUG ] 19:58:56 DEBUG } 19:58:56 DEBUG } 19:58:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639537" 19:58:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639537" 19:58:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639537" 19:58:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639537" 19:58:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639537" 19:58:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639537" 19:58:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639537" 19:58:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639537" 19:58:57 INFO [http_cmd]: http status code OK 19:58:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639537" 19:58:57 DEBUG --- status code --- 19:58:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639537" 19:58:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639537" 19:58:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639537" 19:58:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:57 INFO [http_cmd]: http status code OK 19:58:57 INFO [http_cmd]: http status code OK 19:58:57 INFO [http_cmd]: http status code OK 19:58:57 INFO [http_cmd]: http status code OK 19:58:57 INFO [http_cmd]: http status code OK 19:58:57 INFO [http_cmd]: http status code OK 19:58:57 DEBUG --- status code --- 19:58:57 DEBUG --- http response --- 19:58:57 DEBUG --- status code --- 19:58:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:57 DEBUG --- status code --- 19:58:57 DEBUG --- status code --- 19:58:57 INFO [http_cmd]: http status code OK 19:58:57 DEBUG --- status code --- 19:58:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:57 INFO [http_cmd]: http status code OK 19:58:57 INFO [http_cmd]: http status code OK 19:58:57 DEBUG --- status code --- 19:58:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:58:57 DEBUG --- http response --- 19:58:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:57 DEBUG --- status code --- 19:58:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:57 INFO [http_cmd]: http status code OK 19:58:57 DEBUG --- http response --- 19:58:57 DEBUG --- status code --- 19:58:57 DEBUG --- status code --- 19:58:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:57 DEBUG ----- output ----- 19:58:57 DEBUG --- http response --- 19:58:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:57 DEBUG --- http response --- 19:58:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:57 DEBUG --- http response --- 19:58:57 DEBUG --- status code --- 19:58:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:57 DEBUG --- http response --- 19:58:57 DEBUG { 19:58:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:57 INFO [http_cmd]: http status code OK 19:58:57 DEBUG ----- output ----- 19:58:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:57 DEBUG --- http response --- 19:58:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:57 DEBUG ----- output ----- 19:58:57 DEBUG --- http response --- 19:58:57 DEBUG --- http response --- 19:58:57 DEBUG "status": "success", 19:58:57 DEBUG ----- output ----- 19:58:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:57 DEBUG --- status code --- 19:58:57 DEBUG { 19:58:57 DEBUG ----- output ----- 19:58:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:58:57 DEBUG ----- output ----- 19:58:57 DEBUG --- http response --- 19:58:57 DEBUG { 19:58:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:58:57 DEBUG "data": { 19:58:57 DEBUG { 19:58:57 DEBUG ----- output ----- 19:58:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:58:57 DEBUG "status": "success", 19:58:57 DEBUG { 19:58:57 DEBUG ----- output ----- 19:58:57 DEBUG { 19:58:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:58:57 DEBUG "status": "success", 19:58:57 DEBUG ----- output ----- 19:58:57 DEBUG ----- output ----- 19:58:57 DEBUG "resultType": "vector", 19:58:57 DEBUG "status": "success", 19:58:57 DEBUG { 19:58:57 DEBUG --- http response --- 19:58:57 DEBUG "data": { 19:58:57 DEBUG "status": "success", 19:58:57 DEBUG { 19:58:57 DEBUG "status": "success", 19:58:57 DEBUG ----- output ----- 19:58:57 DEBUG "data": { 19:58:57 DEBUG { 19:58:57 DEBUG { 19:58:57 DEBUG "result": [ 19:58:57 DEBUG "data": { 19:58:57 DEBUG "status": "success", 19:58:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:58:57 DEBUG "resultType": "vector", 19:58:57 DEBUG "data": { 19:58:57 DEBUG "status": "success", 19:58:57 DEBUG "data": { 19:58:57 DEBUG { 19:58:57 DEBUG "resultType": "vector", 19:58:57 DEBUG "status": "success", 19:58:57 DEBUG "status": "success", 19:58:57 DEBUG { 19:58:57 DEBUG "resultType": "vector", 19:58:57 DEBUG "data": { 19:58:57 DEBUG ----- output ----- 19:58:57 DEBUG "result": [ 19:58:57 DEBUG "resultType": "vector", 19:58:57 DEBUG "data": { 19:58:57 DEBUG "resultType": "vector", 19:58:57 DEBUG "status": "success", 19:58:57 DEBUG "result": [ 19:58:57 DEBUG "data": { 19:58:57 DEBUG "resultType": "vector", 19:58:57 DEBUG "metric": { 19:58:57 DEBUG "result": [ 19:58:57 DEBUG "resultType": "vector", 19:58:57 DEBUG { 19:58:57 DEBUG { 19:58:57 DEBUG "result": [ 19:58:57 DEBUG "resultType": "vector", 19:58:57 DEBUG "result": [ 19:58:57 DEBUG "data": { 19:58:57 DEBUG { 19:58:57 DEBUG "result": [ 19:58:57 DEBUG "data": { 19:58:57 DEBUG "pod": "ds-cts-0" 19:58:57 DEBUG { 19:58:57 DEBUG "result": [ 19:58:57 DEBUG "status": "success", 19:58:57 DEBUG "metric": { 19:58:57 DEBUG { 19:58:57 DEBUG "result": [ 19:58:57 DEBUG { 19:58:57 DEBUG "resultType": "vector", 19:58:57 DEBUG "metric": { 19:58:57 DEBUG { 19:58:57 DEBUG "resultType": "vector", 19:58:57 DEBUG }, 19:58:57 DEBUG "metric": { 19:58:57 DEBUG { 19:58:57 DEBUG "data": { 19:58:57 DEBUG "instance": "10.142.0.107:9100" 19:58:57 DEBUG "metric": { 19:58:57 DEBUG { 19:58:57 DEBUG "metric": { 19:58:57 DEBUG "instance": "10.142.0.107:9100" 19:58:57 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:57 DEBUG "metric": { 19:58:57 DEBUG "result": [ 19:58:57 DEBUG "value": [ 19:58:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:58:57 DEBUG "metric": { 19:58:57 DEBUG "resultType": "vector", 19:58:57 DEBUG }, 19:58:57 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:57 DEBUG "metric": { 19:58:57 DEBUG }, 19:58:57 DEBUG "result": [ 19:58:57 DEBUG }, 19:58:57 DEBUG "instance": "10.142.0.107:9100" 19:58:57 DEBUG { 19:58:57 DEBUG 19:58:57 DEBUG }, 19:58:57 DEBUG "instance": "10.142.0.107:9100" 19:58:57 DEBUG "result": [ 19:58:57 DEBUG "value": [ 19:58:57 DEBUG }, 19:58:57 DEBUG "pod": "ds-cts-0" 19:58:57 DEBUG "value": [ 19:58:57 DEBUG { 19:58:57 DEBUG "value": [ 19:58:57 DEBUG }, 19:58:57 DEBUG "metric": { 19:58:57 DEBUG 19:58:57 DEBUG "value": [ 19:58:57 DEBUG }, 19:58:57 DEBUG { 19:58:57 DEBUG 19:58:57 DEBUG "value": [ 19:58:57 DEBUG }, 19:58:57 DEBUG 19:58:57 DEBUG "metric": { 19:58:57 DEBUG 19:58:57 DEBUG "value": [ 19:58:57 DEBUG "pod": "ds-idrepo-0" 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG "value": [ 19:58:57 DEBUG "metric": { 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG "value": [ 19:58:57 DEBUG 19:58:57 DEBUG "pod": "am-767f6b67b5-9zw22" 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG }, 19:58:57 DEBUG [...] 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG "pod": "ds-idrepo-0" 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG }, 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG "value": [ 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG [...] 19:58:57 DEBUG }, 19:58:57 DEBUG [...] 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG [...] 19:58:57 DEBUG "value": [ 19:58:57 DEBUG [...] 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG "value": [ 19:58:57 DEBUG 19:58:57 DEBUG [...] 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG [...] 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG [...] 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG "pod": "ds-cts-2" 19:58:57 DEBUG [...] 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG "instance": "10.142.0.43:9100" 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG [...] 19:58:57 DEBUG }, 19:58:57 DEBUG 19:58:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:58:57 DEBUG 19:58:57 DEBUG "instance": "10.142.0.43:9100" 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG }, 19:58:57 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:57 DEBUG 19:58:57 DEBUG 19:58:57 DEBUG "value": [ 19:58:57 DEBUG 19:58:57 DEBUG }, 19:58:57 DEBUG [...] 19:58:57 DEBUG }, 19:58:57 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:57 DEBUG 19:58:57 DEBUG [...] 19:58:57 DEBUG "value": [ 19:58:57 DEBUG }, 19:58:57 DEBUG "instance": "10.142.0.43:9100" 19:58:57 DEBUG 19:58:57 DEBUG 1713639537, 19:58:57 DEBUG 19:58:57 DEBUG "value": [ 19:58:57 DEBUG 19:58:57 DEBUG "value": [ 19:58:57 DEBUG }, 19:58:57 DEBUG "pod": "ds-cts-2" 19:58:57 DEBUG 19:58:57 DEBUG 1713639537, 19:58:57 DEBUG "value": [ 19:58:57 DEBUG }, 19:58:57 DEBUG 19:58:57 DEBUG "0.00001877111111111111" 19:58:57 DEBUG "instance": "10.142.0.43:9100" 19:58:57 DEBUG 1713639537, 19:58:57 DEBUG 19:58:57 DEBUG 1713639537, 19:58:57 DEBUG "value": [ 19:58:57 DEBUG }, 19:58:57 DEBUG 19:58:57 DEBUG "0.07733333333332933" 19:58:57 DEBUG 1713639537, 19:58:57 DEBUG "value": [ 19:58:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:58:57 DEBUG ] 19:58:57 DEBUG } 19:58:57 DEBUG "920547328" 19:58:57 DEBUG 19:58:57 DEBUG "72621733.23333333" 19:58:57 DEBUG 1713639537, 19:58:57 DEBUG "value": [ 19:58:57 DEBUG 19:58:57 DEBUG ] 19:58:57 DEBUG "0" 19:58:57 DEBUG 1713639537, 19:58:57 DEBUG }, 19:58:57 DEBUG }, 19:58:57 DEBUG ] 19:58:57 DEBUG ] 19:58:57 DEBUG "pod": "ds-idrepo-1" 19:58:57 DEBUG ] 19:58:57 DEBUG "429494" 19:58:57 DEBUG 1713639537, 19:58:57 DEBUG "pod": "am-767f6b67b5-vbltz" 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG "72625668.73333333" 19:58:57 DEBUG "value": [ 19:58:57 DEBUG "value": [ 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG }, 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG "30" 19:58:57 DEBUG }, 19:58:57 DEBUG ] 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG 1713639537, 19:58:57 DEBUG 1713639537, 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG "value": [ 19:58:57 DEBUG ] 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG "value": [ 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG } 19:58:57 DEBUG "0" 19:58:57 DEBUG "0.16480000000000247" 19:58:57 DEBUG } 19:58:57 DEBUG 1713639537, 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG } 19:58:57 DEBUG 1713639537, 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG ] 19:58:57 DEBUG ] 19:58:57 DEBUG } 19:58:57 DEBUG "0" 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG "0.157" 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG ] 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG } 19:58:57 DEBUG ] 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG } 19:58:57 DEBUG } 19:59:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639592" 19:59:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639592" 19:59:52 ERROR [http_cmd]: ERROR 19:59:52 INFO [http_cmd]: http status code OK 19:59:52 ERROR --- status code --- 19:59:52 DEBUG --- status code --- 19:59:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 19:59:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:52 ERROR --- http response --- 19:59:52 DEBUG --- http response --- 19:59:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 19:59:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 19:59:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 19:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639594" 19:59:54 INFO [http_cmd]: http status code OK 19:59:54 DEBUG --- status code --- 19:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:54 DEBUG --- http response --- 19:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:54 DEBUG ----- output ----- 19:59:54 DEBUG { 19:59:54 DEBUG "status": "success", 19:59:54 DEBUG "data": { 19:59:54 DEBUG "resultType": "vector", 19:59:54 DEBUG "result": [ 19:59:54 DEBUG { 19:59:54 DEBUG "metric": { 19:59:54 DEBUG "pod": "ds-idrepo-0" 19:59:54 DEBUG }, 19:59:54 DEBUG "value": [ 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG [...] 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG "pod": "ds-idrepo-2" 19:59:54 DEBUG }, 19:59:54 DEBUG "value": [ 19:59:54 DEBUG 1713639594, 19:59:54 DEBUG "0" 19:59:54 DEBUG ] 19:59:54 DEBUG } 19:59:54 DEBUG ] 19:59:54 DEBUG } 19:59:54 DEBUG } 19:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639594" 19:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639594" 19:59:54 INFO [http_cmd]: http status code OK 19:59:54 DEBUG --- status code --- 19:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:54 DEBUG --- http response --- 19:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:54 DEBUG ----- output ----- 19:59:54 DEBUG { 19:59:54 DEBUG "status": "success", 19:59:54 DEBUG "data": { 19:59:54 DEBUG "resultType": "vector", 19:59:54 DEBUG "result": [ 19:59:54 DEBUG { 19:59:54 DEBUG "metric": { 19:59:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:54 DEBUG }, 19:59:54 DEBUG "value": [ 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG [...] 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:54 DEBUG }, 19:59:54 DEBUG "value": [ 19:59:54 DEBUG 1713639594, 19:59:54 DEBUG "2813.366666666667" 19:59:54 DEBUG ] 19:59:54 DEBUG } 19:59:54 DEBUG ] 19:59:54 DEBUG } 19:59:54 DEBUG } 19:59:54 INFO [http_cmd]: http status code OK 19:59:54 DEBUG --- status code --- 19:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:54 DEBUG --- http response --- 19:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:54 DEBUG ----- output ----- 19:59:54 DEBUG { 19:59:54 DEBUG "status": "success", 19:59:54 DEBUG "data": { 19:59:54 DEBUG "resultType": "vector", 19:59:54 DEBUG "result": [ 19:59:54 DEBUG { 19:59:54 DEBUG "metric": { 19:59:54 DEBUG "pod": "ds-idrepo-0" 19:59:54 DEBUG }, 19:59:54 DEBUG "value": [ 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG [...] 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG "pod": "ds-idrepo-2" 19:59:54 DEBUG }, 19:59:54 DEBUG "value": [ 19:59:54 DEBUG 1713639594, 19:59:54 DEBUG "0" 19:59:54 DEBUG ] 19:59:54 DEBUG } 19:59:54 DEBUG ] 19:59:54 DEBUG } 19:59:54 DEBUG } 19:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639594" 19:59:54 INFO [http_cmd]: http status code OK 19:59:54 DEBUG --- status code --- 19:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:54 DEBUG --- http response --- 19:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 19:59:54 DEBUG ----- output ----- 19:59:54 DEBUG { 19:59:54 DEBUG "status": "success", 19:59:54 DEBUG "data": { 19:59:54 DEBUG "resultType": "vector", 19:59:54 DEBUG "result": [ 19:59:54 DEBUG { 19:59:54 DEBUG "metric": { 19:59:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:59:54 DEBUG }, 19:59:54 DEBUG "value": [ 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG [...] 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 19:59:54 DEBUG }, 19:59:54 DEBUG "value": [ 19:59:54 DEBUG 1713639594, 19:59:54 DEBUG "0" 19:59:54 DEBUG ] 19:59:54 DEBUG } 19:59:54 DEBUG ] 19:59:54 DEBUG } 19:59:54 DEBUG } 19:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639594" 19:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639594" 19:59:54 INFO [http_cmd]: http status code OK 19:59:54 DEBUG --- status code --- 19:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:54 DEBUG --- http response --- 19:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:54 INFO [http_cmd]: http status code OK 19:59:54 DEBUG ----- output ----- 19:59:54 DEBUG --- status code --- 19:59:54 DEBUG { 19:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:54 DEBUG --- http response --- 19:59:54 DEBUG "status": "success", 19:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:54 DEBUG "data": { 19:59:54 DEBUG ----- output ----- 19:59:54 DEBUG "resultType": "vector", 19:59:54 DEBUG { 19:59:54 DEBUG "result": [ 19:59:54 DEBUG "status": "success", 19:59:54 DEBUG { 19:59:54 DEBUG "data": { 19:59:54 DEBUG "metric": { 19:59:54 DEBUG "resultType": "vector", 19:59:54 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:54 DEBUG "result": [ 19:59:54 DEBUG }, 19:59:54 DEBUG { 19:59:54 DEBUG "value": [ 19:59:54 DEBUG "metric": { 19:59:54 DEBUG 19:59:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:59:54 DEBUG 19:59:54 DEBUG }, 19:59:54 DEBUG 19:59:54 DEBUG "value": [ 19:59:54 DEBUG [...] 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG [...] 19:59:54 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:54 DEBUG 19:59:54 DEBUG }, 19:59:54 DEBUG 19:59:54 DEBUG "value": [ 19:59:54 DEBUG 19:59:54 DEBUG 1713639594, 19:59:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:59:54 DEBUG "2833.1" 19:59:54 DEBUG }, 19:59:54 DEBUG ] 19:59:54 DEBUG "value": [ 19:59:54 DEBUG } 19:59:54 DEBUG 1713639594, 19:59:54 DEBUG ] 19:59:54 DEBUG "9.559468663224099" 19:59:54 DEBUG } 19:59:54 DEBUG ] 19:59:54 DEBUG } 19:59:54 DEBUG } 19:59:54 DEBUG ] 19:59:54 DEBUG } 19:59:54 DEBUG } 19:59:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639594" 19:59:54 INFO 19:59:54 INFO [loop_until]: kubectl --namespace=xlou top pods 19:59:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:59:54 INFO [http_cmd]: http status code OK 19:59:54 DEBUG --- status code --- 19:59:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:54 DEBUG --- http response --- 19:59:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:54 DEBUG ----- output ----- 19:59:54 DEBUG { 19:59:54 DEBUG "status": "success", 19:59:54 DEBUG "data": { 19:59:54 DEBUG "resultType": "vector", 19:59:54 DEBUG "result": [ 19:59:54 DEBUG { 19:59:54 DEBUG "metric": { 19:59:54 DEBUG "instance": "10.142.0.107:9100" 19:59:54 DEBUG }, 19:59:54 DEBUG "value": [ 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG [...] 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG 19:59:54 DEBUG "instance": "10.142.0.43:9100" 19:59:54 DEBUG }, 19:59:54 DEBUG "value": [ 19:59:54 DEBUG 1713639594, 19:59:54 DEBUG "3642862216192" 19:59:54 DEBUG ] 19:59:54 DEBUG } 19:59:54 DEBUG ] 19:59:54 DEBUG } 19:59:54 DEBUG } 19:59:55 INFO [loop_until]: OK (rc = 0) 19:59:55 DEBUG --- stdout --- 19:59:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15318m 8650Mi am-767f6b67b5-qcqz2 17190m 9061Mi am-767f6b67b5-vbltz 18020m 8664Mi ds-cts-0 15621m 10382Mi ds-cts-1 13750m 10357Mi ds-cts-2 13205m 10379Mi ds-idrepo-0 7274m 3893Mi ds-idrepo-1 7603m 3882Mi ds-idrepo-2 7475m 3771Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 7m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9476m 791Mi 19:59:55 DEBUG --- stderr --- 19:59:55 DEBUG 19:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639595" 19:59:55 INFO [http_cmd]: http status code OK 19:59:55 DEBUG --- status code --- 19:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:55 DEBUG --- http response --- 19:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:55 DEBUG ----- output ----- 19:59:55 DEBUG { 19:59:55 DEBUG "status": "success", 19:59:55 DEBUG "data": { 19:59:55 DEBUG "resultType": "vector", 19:59:55 DEBUG "result": [ 19:59:55 DEBUG { 19:59:55 DEBUG "metric": { 19:59:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:55 DEBUG }, 19:59:55 DEBUG "value": [ 19:59:55 DEBUG 19:59:55 DEBUG 19:59:55 DEBUG 19:59:55 DEBUG [...] 19:59:55 DEBUG 19:59:55 DEBUG 19:59:55 DEBUG 19:59:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:55 DEBUG }, 19:59:55 DEBUG "value": [ 19:59:55 DEBUG 1713639595, 19:59:55 DEBUG "2832.9666666666667" 19:59:55 DEBUG ] 19:59:55 DEBUG } 19:59:55 DEBUG ] 19:59:55 DEBUG } 19:59:55 DEBUG } 19:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639595" 19:59:55 INFO [http_cmd]: http status code OK 19:59:55 DEBUG --- status code --- 19:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:55 DEBUG --- http response --- 19:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:55 DEBUG ----- output ----- 19:59:55 DEBUG { 19:59:55 DEBUG "status": "success", 19:59:55 DEBUG "data": { 19:59:55 DEBUG "resultType": "vector", 19:59:55 DEBUG "result": [ 19:59:55 DEBUG { 19:59:55 DEBUG "metric": { 19:59:55 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:55 DEBUG }, 19:59:55 DEBUG "value": [ 19:59:55 DEBUG 19:59:55 DEBUG 19:59:55 DEBUG 19:59:55 DEBUG [...] 19:59:55 DEBUG 19:59:55 DEBUG 19:59:55 DEBUG 19:59:55 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:55 DEBUG }, 19:59:55 DEBUG "value": [ 19:59:55 DEBUG 1713639595, 19:59:55 DEBUG "2832.233333333333" 19:59:55 DEBUG ] 19:59:55 DEBUG } 19:59:55 DEBUG ] 19:59:55 DEBUG } 19:59:55 DEBUG } 19:59:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639595" 19:59:55 INFO [http_cmd]: http status code OK 19:59:55 DEBUG --- status code --- 19:59:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:55 DEBUG --- http response --- 19:59:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:55 DEBUG ----- output ----- 19:59:55 DEBUG { 19:59:55 DEBUG "status": "success", 19:59:55 DEBUG "data": { 19:59:55 DEBUG "resultType": "vector", 19:59:55 DEBUG "result": [ 19:59:55 DEBUG { 19:59:55 DEBUG "metric": { 19:59:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:59:55 DEBUG }, 19:59:55 DEBUG "value": [ 19:59:55 DEBUG 19:59:55 DEBUG 19:59:55 DEBUG 19:59:55 DEBUG [...] 19:59:55 DEBUG 19:59:55 DEBUG 19:59:55 DEBUG 19:59:55 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:59:55 DEBUG }, 19:59:55 DEBUG "value": [ 19:59:55 DEBUG 1713639595, 19:59:55 DEBUG "0" 19:59:55 DEBUG ] 19:59:55 DEBUG } 19:59:55 DEBUG ] 19:59:55 DEBUG } 19:59:55 DEBUG } 19:59:56 INFO 19:59:56 INFO [loop_until]: kubectl --namespace=xlou top node 19:59:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 19:59:56 INFO [loop_until]: OK (rc = 0) 19:59:56 DEBUG --- stdout --- 19:59:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6215Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7616m 25% 7651Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16552m 55% 11371Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3342Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 13878m 46% 13560Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5219Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16323m 54% 11943Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15610m 52% 2969Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18948m 63% 10881Mi 9% gke-xlou-cdm-ds-0f601751-j488 8116m 27% 7164Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 12554m 42% 15384Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8287m 27% 7853Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 15684m 52% 13022Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 9319m 58% 6395Mi 10% 19:59:56 DEBUG --- stderr --- 19:59:56 DEBUG 19:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639596" 19:59:56 INFO [http_cmd]: http status code OK 19:59:56 DEBUG --- status code --- 19:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:56 DEBUG --- http response --- 19:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:56 DEBUG ----- output ----- 19:59:56 DEBUG { 19:59:56 DEBUG "status": "success", 19:59:56 DEBUG "data": { 19:59:56 DEBUG "resultType": "vector", 19:59:56 DEBUG "result": [ 19:59:56 DEBUG { 19:59:56 DEBUG "metric": { 19:59:56 DEBUG "pod": "ds-idrepo-0" 19:59:56 DEBUG }, 19:59:56 DEBUG "value": [ 19:59:56 DEBUG 19:59:56 DEBUG 19:59:56 DEBUG 19:59:56 DEBUG [...] 19:59:56 DEBUG 19:59:56 DEBUG 19:59:56 DEBUG 19:59:56 DEBUG "pod": "ds-idrepo-2" 19:59:56 DEBUG }, 19:59:56 DEBUG "value": [ 19:59:56 DEBUG 1713639596, 19:59:56 DEBUG "0" 19:59:56 DEBUG ] 19:59:56 DEBUG } 19:59:56 DEBUG ] 19:59:56 DEBUG } 19:59:56 DEBUG } 19:59:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639596" 19:59:56 INFO [http_cmd]: http status code OK 19:59:56 DEBUG --- status code --- 19:59:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:56 DEBUG --- http response --- 19:59:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:59:56 DEBUG ----- output ----- 19:59:56 DEBUG { 19:59:56 DEBUG "status": "success", 19:59:56 DEBUG "data": { 19:59:56 DEBUG "resultType": "vector", 19:59:56 DEBUG "result": [ 19:59:56 DEBUG { 19:59:56 DEBUG "metric": { 19:59:56 DEBUG "pod": "ds-cts-0" 19:59:56 DEBUG }, 19:59:56 DEBUG "value": [ 19:59:56 DEBUG 19:59:56 DEBUG 19:59:56 DEBUG 19:59:56 DEBUG [...] 19:59:56 DEBUG 19:59:56 DEBUG 19:59:56 DEBUG 19:59:56 DEBUG "pod": "ds-cts-2" 19:59:56 DEBUG }, 19:59:56 DEBUG "value": [ 19:59:56 DEBUG 1713639596, 19:59:56 DEBUG "0" 19:59:56 DEBUG ] 19:59:56 DEBUG } 19:59:56 DEBUG ] 19:59:56 DEBUG } 19:59:56 DEBUG } 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639597" 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 DEBUG --- status code --- 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG { 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG "data": { 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG "result": [ 19:59:57 DEBUG { 19:59:57 DEBUG "metric": { 19:59:57 DEBUG "pod": "ds-idrepo-0" 19:59:57 DEBUG }, 19:59:57 DEBUG "value": [ 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG [...] 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG "pod": "ds-cts-2" 19:59:57 DEBUG }, 19:59:57 DEBUG "value": [ 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG "0" 19:59:57 DEBUG ] 19:59:57 DEBUG } 19:59:57 DEBUG ] 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639597" 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 DEBUG --- status code --- 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG { 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG "data": { 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG "result": [ 19:59:57 DEBUG { 19:59:57 DEBUG "metric": { 19:59:57 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:57 DEBUG }, 19:59:57 DEBUG "value": [ 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG [...] 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:57 DEBUG }, 19:59:57 DEBUG "value": [ 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG "510045" 19:59:57 DEBUG ] 19:59:57 DEBUG } 19:59:57 DEBUG ] 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639597" 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639597" 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639597" 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639597" 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639597" 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639597" 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639597" 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639597" 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639597" 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639597" 19:59:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639597" 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 DEBUG --- status code --- 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 DEBUG --- status code --- 19:59:57 DEBUG --- status code --- 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 DEBUG --- status code --- 19:59:57 DEBUG --- status code --- 19:59:57 DEBUG --- status code --- 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 DEBUG --- status code --- 19:59:57 INFO [http_cmd]: http status code OK 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG --- status code --- 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG --- status code --- 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG --- status code --- 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG { 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 19:59:57 DEBUG --- status code --- 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG { 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG "data": { 19:59:57 DEBUG { 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG { 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 19:59:57 DEBUG { 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 19:59:57 DEBUG { 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG "data": { 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG { 19:59:57 DEBUG --- http response --- 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG { 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG "result": [ 19:59:57 DEBUG "data": { 19:59:57 DEBUG { 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG "data": { 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 19:59:57 DEBUG "data": { 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG { 19:59:57 DEBUG "data": { 19:59:57 DEBUG { 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG "result": [ 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG "data": { 19:59:57 DEBUG ----- output ----- 19:59:57 DEBUG { 19:59:57 DEBUG "data": { 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG "metric": { 19:59:57 DEBUG "result": [ 19:59:57 DEBUG "data": { 19:59:57 DEBUG { 19:59:57 DEBUG "result": [ 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG "status": "success", 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG "data": { 19:59:57 DEBUG "result": [ 19:59:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 19:59:57 DEBUG { 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG "metric": { 19:59:57 DEBUG { 19:59:57 DEBUG "result": [ 19:59:57 DEBUG "result": [ 19:59:57 DEBUG "data": { 19:59:57 DEBUG "result": [ 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG { 19:59:57 DEBUG }, 19:59:57 DEBUG "metric": { 19:59:57 DEBUG "result": [ 19:59:57 DEBUG "pod": "ds-cts-0" 19:59:57 DEBUG "metric": { 19:59:57 DEBUG { 19:59:57 DEBUG { 19:59:57 DEBUG "resultType": "vector", 19:59:57 DEBUG { 19:59:57 DEBUG "result": [ 19:59:57 DEBUG { 19:59:57 DEBUG "value": [ 19:59:57 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:57 DEBUG { 19:59:57 DEBUG }, 19:59:57 DEBUG "instance": "10.142.0.107:9100" 19:59:57 DEBUG "metric": { 19:59:57 DEBUG "metric": { 19:59:57 DEBUG "result": [ 19:59:57 DEBUG "metric": { 19:59:57 DEBUG "metric": { 19:59:57 DEBUG "metric": { 19:59:57 DEBUG 19:59:57 DEBUG }, 19:59:57 DEBUG "value": [ 19:59:57 DEBUG "value": [ 19:59:57 DEBUG }, 19:59:57 DEBUG "pod": "am-767f6b67b5-9zw22" 19:59:57 DEBUG "instance": "10.142.0.107:9100" 19:59:57 DEBUG { 19:59:57 DEBUG "pod": "ds-cts-0" 19:59:57 DEBUG "instance": "10.142.0.107:9100" 19:59:57 DEBUG "instance": "10.142.0.107:9100" 19:59:57 DEBUG 19:59:57 DEBUG "metric": { 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG "value": [ 19:59:57 DEBUG }, 19:59:57 DEBUG }, 19:59:57 DEBUG "metric": { 19:59:57 DEBUG }, 19:59:57 DEBUG }, 19:59:57 DEBUG "value": [ 19:59:57 DEBUG 19:59:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG "value": [ 19:59:57 DEBUG "value": [ 19:59:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 19:59:57 DEBUG "value": [ 19:59:57 DEBUG }, 19:59:57 DEBUG 19:59:57 DEBUG [...] 19:59:57 DEBUG }, 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG }, 19:59:57 DEBUG 19:59:57 DEBUG "value": [ 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG "value": [ 19:59:57 DEBUG [...] 19:59:57 DEBUG [...] 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG "value": [ 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG [...] 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG [...] 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG [...] 19:59:57 DEBUG [...] 19:59:57 DEBUG 19:59:57 DEBUG [...] 19:59:57 DEBUG [...] 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 19:59:57 DEBUG 19:59:57 DEBUG [...] 19:59:57 DEBUG }, 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG [...] 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG }, 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG "value": [ 19:59:57 DEBUG "pod": "ds-cts-2" 19:59:57 DEBUG "instance": "10.142.0.43:9100" 19:59:57 DEBUG 19:59:57 DEBUG "pod": "am-767f6b67b5-vbltz" 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG 19:59:57 DEBUG "value": [ 19:59:57 DEBUG "instance": "10.142.0.43:9100" 19:59:57 DEBUG 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG }, 19:59:57 DEBUG }, 19:59:57 DEBUG }, 19:59:57 DEBUG 19:59:57 DEBUG "pod": "ds-cts-2" 19:59:57 DEBUG 19:59:57 DEBUG "instance": "10.142.0.43:9100" 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG }, 19:59:57 DEBUG 19:59:57 DEBUG "0" 19:59:57 DEBUG "value": [ 19:59:57 DEBUG "value": [ 19:59:57 DEBUG "value": [ 19:59:57 DEBUG 19:59:57 DEBUG }, 19:59:57 DEBUG "instance": "10.142.0.43:9100" 19:59:57 DEBUG }, 19:59:57 DEBUG "928268288" 19:59:57 DEBUG "value": [ 19:59:57 DEBUG "pod": "am-767f6b67b5-qcqz2" 19:59:57 DEBUG ] 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 19:59:57 DEBUG }, 19:59:57 DEBUG }, 19:59:57 DEBUG "value": [ 19:59:57 DEBUG ] 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG }, 19:59:57 DEBUG } 19:59:57 DEBUG "0.00072298" 19:59:57 DEBUG "0.039666666666681755" 19:59:57 DEBUG "0.157" 19:59:57 DEBUG "value": [ 19:59:57 DEBUG "value": [ 19:59:57 DEBUG "value": [ 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG } 19:59:57 DEBUG "0.09290000000001442" 19:59:57 DEBUG "value": [ 19:59:57 DEBUG ] 19:59:57 DEBUG ] 19:59:57 DEBUG ] 19:59:57 DEBUG ] 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG "84928427.63333333" 19:59:57 DEBUG ] 19:59:57 DEBUG ] 19:59:57 DEBUG 1713639597, 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG ] 19:59:57 DEBUG } 19:59:57 DEBUG "29.966666666666665" 19:59:57 DEBUG "0" 19:59:57 DEBUG "84923751.83333333" 19:59:57 DEBUG ] 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG "0" 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG ] 19:59:57 DEBUG ] 19:59:57 DEBUG ] 19:59:57 DEBUG ] 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG ] 19:59:57 DEBUG ] 19:59:57 DEBUG } 19:59:57 DEBUG ] 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG ] 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG ] 19:59:57 DEBUG ] 19:59:57 DEBUG ] 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG ] 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 19:59:57 DEBUG } 20:00:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639652" 20:00:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639652" 20:00:52 ERROR [http_cmd]: ERROR 20:00:52 ERROR --- status code --- 20:00:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:00:52 ERROR --- http response --- 20:00:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:00:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:00:52 INFO [http_cmd]: http status code OK 20:00:52 DEBUG --- status code --- 20:00:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:52 DEBUG --- http response --- 20:00:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639654" 20:00:54 INFO [http_cmd]: http status code OK 20:00:54 DEBUG --- status code --- 20:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:54 DEBUG --- http response --- 20:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:54 DEBUG ----- output ----- 20:00:54 DEBUG { 20:00:54 DEBUG "status": "success", 20:00:54 DEBUG "data": { 20:00:54 DEBUG "resultType": "vector", 20:00:54 DEBUG "result": [ 20:00:54 DEBUG { 20:00:54 DEBUG "metric": { 20:00:54 DEBUG "pod": "ds-idrepo-0" 20:00:54 DEBUG }, 20:00:54 DEBUG "value": [ 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG [...] 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG "pod": "ds-idrepo-2" 20:00:54 DEBUG }, 20:00:54 DEBUG "value": [ 20:00:54 DEBUG 1713639654, 20:00:54 DEBUG "0" 20:00:54 DEBUG ] 20:00:54 DEBUG } 20:00:54 DEBUG ] 20:00:54 DEBUG } 20:00:54 DEBUG } 20:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639654" 20:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639654" 20:00:54 INFO [http_cmd]: http status code OK 20:00:54 DEBUG --- status code --- 20:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:54 DEBUG --- http response --- 20:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:54 DEBUG ----- output ----- 20:00:54 DEBUG { 20:00:54 DEBUG "status": "success", 20:00:54 DEBUG "data": { 20:00:54 DEBUG "resultType": "vector", 20:00:54 DEBUG "result": [ 20:00:54 DEBUG { 20:00:54 DEBUG "metric": { 20:00:54 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:54 DEBUG }, 20:00:54 DEBUG "value": [ 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG [...] 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:54 DEBUG }, 20:00:54 DEBUG "value": [ 20:00:54 DEBUG 1713639654, 20:00:54 DEBUG "2837.7" 20:00:54 DEBUG ] 20:00:54 DEBUG } 20:00:54 DEBUG ] 20:00:54 DEBUG } 20:00:54 DEBUG } 20:00:54 INFO [http_cmd]: http status code OK 20:00:54 DEBUG --- status code --- 20:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:54 DEBUG --- http response --- 20:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:54 DEBUG ----- output ----- 20:00:54 DEBUG { 20:00:54 DEBUG "status": "success", 20:00:54 DEBUG "data": { 20:00:54 DEBUG "resultType": "vector", 20:00:54 DEBUG "result": [ 20:00:54 DEBUG { 20:00:54 DEBUG "metric": { 20:00:54 DEBUG "pod": "ds-idrepo-0" 20:00:54 DEBUG }, 20:00:54 DEBUG "value": [ 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG [...] 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG "pod": "ds-idrepo-2" 20:00:54 DEBUG }, 20:00:54 DEBUG "value": [ 20:00:54 DEBUG 1713639654, 20:00:54 DEBUG "0" 20:00:54 DEBUG ] 20:00:54 DEBUG } 20:00:54 DEBUG ] 20:00:54 DEBUG } 20:00:54 DEBUG } 20:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639654" 20:00:54 INFO [http_cmd]: http status code OK 20:00:54 DEBUG --- status code --- 20:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:54 DEBUG --- http response --- 20:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:54 DEBUG ----- output ----- 20:00:54 DEBUG { 20:00:54 DEBUG "status": "success", 20:00:54 DEBUG "data": { 20:00:54 DEBUG "resultType": "vector", 20:00:54 DEBUG "result": [ 20:00:54 DEBUG { 20:00:54 DEBUG "metric": { 20:00:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:00:54 DEBUG }, 20:00:54 DEBUG "value": [ 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG [...] 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:00:54 DEBUG }, 20:00:54 DEBUG "value": [ 20:00:54 DEBUG 1713639654, 20:00:54 DEBUG "0" 20:00:54 DEBUG ] 20:00:54 DEBUG } 20:00:54 DEBUG ] 20:00:54 DEBUG } 20:00:54 DEBUG } 20:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639654" 20:00:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639654" 20:00:54 INFO [http_cmd]: http status code OK 20:00:54 DEBUG --- status code --- 20:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:54 DEBUG --- http response --- 20:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:54 DEBUG ----- output ----- 20:00:54 DEBUG { 20:00:54 DEBUG "status": "success", 20:00:54 DEBUG "data": { 20:00:54 DEBUG "resultType": "vector", 20:00:54 DEBUG "result": [ 20:00:54 DEBUG { 20:00:54 DEBUG "metric": { 20:00:54 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:54 DEBUG }, 20:00:54 DEBUG "value": [ 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG [...] 20:00:54 DEBUG 20:00:54 INFO [http_cmd]: http status code OK 20:00:54 DEBUG 20:00:54 DEBUG --- status code --- 20:00:54 DEBUG 20:00:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:54 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:54 DEBUG --- http response --- 20:00:54 DEBUG }, 20:00:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:54 DEBUG "value": [ 20:00:54 DEBUG ----- output ----- 20:00:54 DEBUG { 20:00:54 DEBUG "status": "success", 20:00:54 DEBUG "data": { 20:00:54 DEBUG "resultType": "vector", 20:00:54 DEBUG "result": [ 20:00:54 DEBUG { 20:00:54 DEBUG "metric": { 20:00:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:00:54 DEBUG }, 20:00:54 DEBUG "value": [ 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG [...] 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG 20:00:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:00:54 DEBUG }, 20:00:54 DEBUG "value": [ 20:00:54 DEBUG 1713639654, 20:00:54 DEBUG "9.643802107751057" 20:00:54 DEBUG ] 20:00:54 DEBUG } 20:00:54 DEBUG ] 20:00:54 DEBUG } 20:00:54 DEBUG } 20:00:54 DEBUG 1713639654, 20:00:54 DEBUG "2843.4333333333334" 20:00:54 DEBUG ] 20:00:54 DEBUG } 20:00:54 DEBUG ] 20:00:54 DEBUG } 20:00:54 DEBUG } 20:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639655" 20:00:55 INFO [http_cmd]: http status code OK 20:00:55 DEBUG --- status code --- 20:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:55 DEBUG --- http response --- 20:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:55 DEBUG ----- output ----- 20:00:55 DEBUG { 20:00:55 DEBUG "status": "success", 20:00:55 DEBUG "data": { 20:00:55 DEBUG "resultType": "vector", 20:00:55 DEBUG "result": [ 20:00:55 DEBUG { 20:00:55 DEBUG "metric": { 20:00:55 DEBUG "instance": "10.142.0.107:9100" 20:00:55 DEBUG }, 20:00:55 DEBUG "value": [ 20:00:55 DEBUG 20:00:55 DEBUG 20:00:55 DEBUG 20:00:55 DEBUG [...] 20:00:55 DEBUG 20:00:55 DEBUG 20:00:55 DEBUG 20:00:55 DEBUG "instance": "10.142.0.43:9100" 20:00:55 DEBUG }, 20:00:55 DEBUG "value": [ 20:00:55 DEBUG 1713639655, 20:00:55 DEBUG "3646557891584" 20:00:55 DEBUG ] 20:00:55 DEBUG } 20:00:55 DEBUG ] 20:00:55 DEBUG } 20:00:55 DEBUG } 20:00:55 INFO 20:00:55 INFO [loop_until]: kubectl --namespace=xlou top pods 20:00:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:00:55 INFO [loop_until]: OK (rc = 0) 20:00:55 DEBUG --- stdout --- 20:00:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17984m 8654Mi am-767f6b67b5-qcqz2 18121m 9063Mi am-767f6b67b5-vbltz 20361m 8668Mi ds-cts-0 20520m 10789Mi ds-cts-1 18551m 10751Mi ds-cts-2 20222m 10808Mi ds-idrepo-0 7923m 3893Mi ds-idrepo-1 7913m 3883Mi ds-idrepo-2 7821m 3771Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 18m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9209m 793Mi 20:00:55 DEBUG --- stderr --- 20:00:55 DEBUG 20:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639655" 20:00:55 INFO [http_cmd]: http status code OK 20:00:55 DEBUG --- status code --- 20:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:55 DEBUG --- http response --- 20:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:55 DEBUG ----- output ----- 20:00:55 DEBUG { 20:00:55 DEBUG "status": "success", 20:00:55 DEBUG "data": { 20:00:55 DEBUG "resultType": "vector", 20:00:55 DEBUG "result": [ 20:00:55 DEBUG { 20:00:55 DEBUG "metric": { 20:00:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:55 DEBUG }, 20:00:55 DEBUG "value": [ 20:00:55 DEBUG 20:00:55 DEBUG 20:00:55 DEBUG 20:00:55 DEBUG [...] 20:00:55 DEBUG 20:00:55 DEBUG 20:00:55 DEBUG 20:00:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:55 DEBUG }, 20:00:55 DEBUG "value": [ 20:00:55 DEBUG 1713639655, 20:00:55 DEBUG "2843.366666666667" 20:00:55 DEBUG ] 20:00:55 DEBUG } 20:00:55 DEBUG ] 20:00:55 DEBUG } 20:00:55 DEBUG } 20:00:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639655" 20:00:55 INFO [http_cmd]: http status code OK 20:00:55 DEBUG --- status code --- 20:00:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:55 DEBUG --- http response --- 20:00:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:55 DEBUG ----- output ----- 20:00:55 DEBUG { 20:00:55 DEBUG "status": "success", 20:00:55 DEBUG "data": { 20:00:55 DEBUG "resultType": "vector", 20:00:55 DEBUG "result": [ 20:00:55 DEBUG { 20:00:55 DEBUG "metric": { 20:00:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:55 DEBUG }, 20:00:55 DEBUG "value": [ 20:00:55 DEBUG 20:00:55 DEBUG 20:00:55 DEBUG 20:00:55 DEBUG [...] 20:00:55 DEBUG 20:00:55 DEBUG 20:00:55 DEBUG 20:00:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:55 DEBUG }, 20:00:55 DEBUG "value": [ 20:00:55 DEBUG 1713639655, 20:00:55 DEBUG "2842.3" 20:00:55 DEBUG ] 20:00:55 DEBUG } 20:00:55 DEBUG ] 20:00:55 DEBUG } 20:00:55 DEBUG } 20:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639656" 20:00:56 INFO [http_cmd]: http status code OK 20:00:56 DEBUG --- status code --- 20:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:56 DEBUG --- http response --- 20:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:00:56 DEBUG ----- output ----- 20:00:56 DEBUG { 20:00:56 DEBUG "status": "success", 20:00:56 DEBUG "data": { 20:00:56 DEBUG "resultType": "vector", 20:00:56 DEBUG "result": [ 20:00:56 DEBUG { 20:00:56 DEBUG "metric": { 20:00:56 DEBUG "pod": "ds-idrepo-1" 20:00:56 DEBUG }, 20:00:56 DEBUG "value": [ 20:00:56 DEBUG 20:00:56 DEBUG 20:00:56 DEBUG 20:00:56 DEBUG [...] 20:00:56 DEBUG 20:00:56 DEBUG 20:00:56 DEBUG 20:00:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:00:56 DEBUG }, 20:00:56 DEBUG "value": [ 20:00:56 DEBUG 1713639656, 20:00:56 DEBUG "0.00002959299711741941" 20:00:56 DEBUG ] 20:00:56 DEBUG } 20:00:56 DEBUG ] 20:00:56 DEBUG } 20:00:56 DEBUG } 20:00:56 INFO 20:00:56 INFO [loop_until]: kubectl --namespace=xlou top node 20:00:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639656" 20:00:56 INFO [http_cmd]: http status code OK 20:00:56 DEBUG --- status code --- 20:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:56 DEBUG --- http response --- 20:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:56 DEBUG ----- output ----- 20:00:56 DEBUG { 20:00:56 DEBUG "status": "success", 20:00:56 DEBUG "data": { 20:00:56 DEBUG "resultType": "vector", 20:00:56 DEBUG "result": [ 20:00:56 DEBUG { 20:00:56 DEBUG "metric": { 20:00:56 DEBUG "pod": "ds-idrepo-0" 20:00:56 DEBUG }, 20:00:56 DEBUG "value": [ 20:00:56 DEBUG 20:00:56 DEBUG 20:00:56 DEBUG 20:00:56 DEBUG [...] 20:00:56 DEBUG 20:00:56 DEBUG 20:00:56 DEBUG 20:00:56 DEBUG "pod": "ds-idrepo-2" 20:00:56 DEBUG }, 20:00:56 DEBUG "value": [ 20:00:56 DEBUG 1713639656, 20:00:56 DEBUG "0" 20:00:56 DEBUG ] 20:00:56 DEBUG } 20:00:56 DEBUG ] 20:00:56 DEBUG } 20:00:56 DEBUG } 20:00:56 INFO [loop_until]: OK (rc = 0) 20:00:56 DEBUG --- stdout --- 20:00:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6223Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8190m 27% 7649Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18201m 60% 11372Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3341Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20285m 67% 13921Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 263m 0% 5219Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18426m 61% 11956Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15678m 52% 2968Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20863m 69% 10881Mi 9% gke-xlou-cdm-ds-0f601751-j488 8721m 29% 7172Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18450m 61% 15799Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8600m 28% 7856Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20710m 69% 13486Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9863m 62% 6397Mi 10% 20:00:56 DEBUG --- stderr --- 20:00:56 DEBUG 20:00:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639656" 20:00:56 INFO [http_cmd]: http status code OK 20:00:56 DEBUG --- status code --- 20:00:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:56 DEBUG --- http response --- 20:00:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:00:56 DEBUG ----- output ----- 20:00:56 DEBUG { 20:00:56 DEBUG "status": "success", 20:00:56 DEBUG "data": { 20:00:56 DEBUG "resultType": "vector", 20:00:56 DEBUG "result": [ 20:00:56 DEBUG { 20:00:56 DEBUG "metric": { 20:00:56 DEBUG "pod": "ds-cts-0" 20:00:56 DEBUG }, 20:00:56 DEBUG "value": [ 20:00:56 DEBUG 20:00:56 DEBUG 20:00:56 DEBUG 20:00:56 DEBUG [...] 20:00:56 DEBUG 20:00:56 DEBUG 20:00:56 DEBUG 20:00:56 DEBUG "pod": "ds-cts-2" 20:00:56 DEBUG }, 20:00:56 DEBUG "value": [ 20:00:56 DEBUG 1713639656, 20:00:56 DEBUG "0" 20:00:56 DEBUG ] 20:00:56 DEBUG } 20:00:56 DEBUG ] 20:00:56 DEBUG } 20:00:56 DEBUG } 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639657" 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 DEBUG --- status code --- 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 DEBUG --- http response --- 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG { 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG "data": { 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG "result": [ 20:00:57 DEBUG { 20:00:57 DEBUG "metric": { 20:00:57 DEBUG "pod": "ds-idrepo-0" 20:00:57 DEBUG }, 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG [...] 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG "pod": "ds-cts-2" 20:00:57 DEBUG }, 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG "0" 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639657" 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 DEBUG --- status code --- 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 DEBUG --- http response --- 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG { 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG "data": { 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG "result": [ 20:00:57 DEBUG { 20:00:57 DEBUG "metric": { 20:00:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:57 DEBUG }, 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG [...] 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:57 DEBUG }, 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG "510525.5" 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639657" 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639657" 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639657" 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639657" 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639657" 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639657" 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639657" 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639657" 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639657" 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639657" 20:00:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639657" 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 DEBUG --- status code --- 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 DEBUG --- status code --- 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 DEBUG --- status code --- 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 DEBUG --- status code --- 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 DEBUG --- http response --- 20:00:57 DEBUG --- status code --- 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 DEBUG --- http response --- 20:00:57 DEBUG --- status code --- 20:00:57 DEBUG --- status code --- 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 DEBUG --- http response --- 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 DEBUG --- status code --- 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 DEBUG --- http response --- 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 DEBUG --- status code --- 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:00:57 INFO [http_cmd]: http status code OK 20:00:57 DEBUG --- status code --- 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG --- http response --- 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG { 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG --- http response --- 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG --- status code --- 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 DEBUG --- http response --- 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 DEBUG "data": { 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:57 DEBUG { 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG --- http response --- 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG { 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG "data": { 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG { 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG "data": { 20:00:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:00:57 DEBUG "result": [ 20:00:57 DEBUG --- http response --- 20:00:57 DEBUG { 20:00:57 DEBUG --- http response --- 20:00:57 DEBUG "data": { 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG { 20:00:57 DEBUG { 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG --- http response --- 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG { 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG "result": [ 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG "result": [ 20:00:57 DEBUG { 20:00:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG ----- output ----- 20:00:57 DEBUG "data": { 20:00:57 DEBUG { 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG { 20:00:57 DEBUG "data": { 20:00:57 DEBUG "data": { 20:00:57 DEBUG { 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG { 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG { 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG "result": [ 20:00:57 DEBUG "metric": { 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG "metric": { 20:00:57 DEBUG "data": { 20:00:57 DEBUG "data": { 20:00:57 DEBUG "metric": { 20:00:57 DEBUG "status": "success", 20:00:57 DEBUG "result": [ 20:00:57 DEBUG { 20:00:57 DEBUG { 20:00:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:57 DEBUG "result": [ 20:00:57 DEBUG "result": [ 20:00:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG "instance": "10.142.0.107:9100" 20:00:57 DEBUG "data": { 20:00:57 DEBUG "metric": { 20:00:57 DEBUG "metric": { 20:00:57 DEBUG "data": { 20:00:57 DEBUG }, 20:00:57 DEBUG { 20:00:57 DEBUG { 20:00:57 DEBUG }, 20:00:57 DEBUG "result": [ 20:00:57 DEBUG "result": [ 20:00:57 DEBUG }, 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 20:00:57 DEBUG "pod": "ds-cts-0" 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG "result": [ 20:00:57 DEBUG "metric": { 20:00:57 DEBUG "metric": { 20:00:57 DEBUG "value": [ 20:00:57 DEBUG { 20:00:57 DEBUG { 20:00:57 DEBUG "resultType": "vector", 20:00:57 DEBUG 20:00:57 DEBUG "instance": "10.142.0.107:9100" 20:00:57 DEBUG }, 20:00:57 DEBUG "value": [ 20:00:57 DEBUG { 20:00:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:00:57 DEBUG }, 20:00:57 DEBUG 20:00:57 DEBUG "metric": { 20:00:57 DEBUG "metric": { 20:00:57 DEBUG "pod": "ds-idrepo-1" 20:00:57 DEBUG 20:00:57 DEBUG }, 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 20:00:57 DEBUG "metric": { 20:00:57 DEBUG "value": [ 20:00:57 DEBUG "instance": "10.142.0.107:9100" 20:00:57 DEBUG 20:00:57 DEBUG "pod": "ds-cts-0" 20:00:57 DEBUG }, 20:00:57 DEBUG "result": [ 20:00:57 DEBUG [...] 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG "pod": "ds-idrepo-1" 20:00:57 DEBUG 20:00:57 DEBUG }, 20:00:57 DEBUG 20:00:57 DEBUG }, 20:00:57 DEBUG "value": [ 20:00:57 DEBUG { 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG "value": [ 20:00:57 DEBUG [...] 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG "metric": { 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG }, 20:00:57 DEBUG [...] 20:00:57 DEBUG [...] 20:00:57 DEBUG 20:00:57 DEBUG [...] 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG "instance": "10.142.0.107:9100" 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 20:00:57 DEBUG "instance": "10.142.0.43:9100" 20:00:57 DEBUG [...] 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:57 DEBUG [...] 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG }, 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG }, 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG [...] 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 20:00:57 DEBUG "instance": "10.142.0.43:9100" 20:00:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:00:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:00:57 DEBUG [...] 20:00:57 DEBUG "value": [ 20:00:57 DEBUG "pod": "ds-cts-2" 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG }, 20:00:57 DEBUG }, 20:00:57 DEBUG }, 20:00:57 DEBUG }, 20:00:57 DEBUG 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG }, 20:00:57 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:00:57 DEBUG 20:00:57 DEBUG [...] 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 20:00:57 DEBUG "0.157" 20:00:57 DEBUG "value": [ 20:00:57 DEBUG }, 20:00:57 DEBUG "instance": "10.142.0.43:9100" 20:00:57 DEBUG "0.060333333333340976" 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG "value": [ 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG "value": [ 20:00:57 DEBUG ] 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG "0.010191699999999998" 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG ] 20:00:57 DEBUG 20:00:57 DEBUG 20:00:57 DEBUG "0" 20:00:57 DEBUG 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG "931893248" 20:00:57 DEBUG ] 20:00:57 DEBUG ] 20:00:57 DEBUG "0" 20:00:57 DEBUG 20:00:57 DEBUG } 20:00:57 DEBUG 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG "0.1704000000000112" 20:00:57 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:00:57 DEBUG } 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG }, 20:00:57 DEBUG "pod": "ds-cts-2" 20:00:57 DEBUG } 20:00:57 DEBUG [...] 20:00:57 DEBUG ] 20:00:57 DEBUG ] 20:00:57 DEBUG }, 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG ] 20:00:57 DEBUG ] 20:00:57 DEBUG "value": [ 20:00:57 DEBUG }, 20:00:57 DEBUG 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG ] 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG "value": [ 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG "86355247.69999999" 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG "82992.69469331496" 20:00:57 DEBUG ] 20:00:57 DEBUG "31.53333333333333" 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG ] 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG ] 20:00:57 DEBUG "instance": "10.142.0.43:9100" 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG ] 20:00:57 DEBUG }, 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG } 20:00:57 DEBUG "value": [ 20:00:57 DEBUG 1713639657, 20:00:57 DEBUG "86357971.53333333" 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG ] 20:00:57 DEBUG } 20:00:57 DEBUG } 20:01:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639712" 20:01:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639712" 20:01:52 ERROR [http_cmd]: ERROR 20:01:52 ERROR --- status code --- 20:01:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:01:52 ERROR --- http response --- 20:01:52 INFO [http_cmd]: http status code OK 20:01:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:01:52 DEBUG --- status code --- 20:01:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:01:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:52 DEBUG --- http response --- 20:01:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:01:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639714" 20:01:54 INFO [http_cmd]: http status code OK 20:01:54 DEBUG --- status code --- 20:01:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:54 DEBUG --- http response --- 20:01:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:54 DEBUG ----- output ----- 20:01:54 DEBUG { 20:01:54 DEBUG "status": "success", 20:01:54 DEBUG "data": { 20:01:54 DEBUG "resultType": "vector", 20:01:54 DEBUG "result": [ 20:01:54 DEBUG { 20:01:54 DEBUG "metric": { 20:01:54 DEBUG "pod": "ds-idrepo-0" 20:01:54 DEBUG }, 20:01:54 DEBUG "value": [ 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG [...] 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG "pod": "ds-idrepo-2" 20:01:54 DEBUG }, 20:01:54 DEBUG "value": [ 20:01:54 DEBUG 1713639714, 20:01:54 DEBUG "0" 20:01:54 DEBUG ] 20:01:54 DEBUG } 20:01:54 DEBUG ] 20:01:54 DEBUG } 20:01:54 DEBUG } 20:01:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639714" 20:01:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639714" 20:01:54 INFO [http_cmd]: http status code OK 20:01:54 DEBUG --- status code --- 20:01:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:54 DEBUG --- http response --- 20:01:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:54 DEBUG ----- output ----- 20:01:54 DEBUG { 20:01:54 DEBUG "status": "success", 20:01:54 DEBUG "data": { 20:01:54 DEBUG "resultType": "vector", 20:01:54 DEBUG "result": [ 20:01:54 DEBUG { 20:01:54 DEBUG "metric": { 20:01:54 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:54 DEBUG }, 20:01:54 DEBUG "value": [ 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG [...] 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:54 DEBUG }, 20:01:54 DEBUG "value": [ 20:01:54 DEBUG 1713639714, 20:01:54 DEBUG "2843.6666666666665" 20:01:54 DEBUG ] 20:01:54 DEBUG } 20:01:54 DEBUG ] 20:01:54 DEBUG } 20:01:54 DEBUG } 20:01:54 INFO [http_cmd]: http status code OK 20:01:54 DEBUG --- status code --- 20:01:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:54 DEBUG --- http response --- 20:01:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:54 DEBUG ----- output ----- 20:01:54 DEBUG { 20:01:54 DEBUG "status": "success", 20:01:54 DEBUG "data": { 20:01:54 DEBUG "resultType": "vector", 20:01:54 DEBUG "result": [ 20:01:54 DEBUG { 20:01:54 DEBUG "metric": { 20:01:54 DEBUG "pod": "ds-idrepo-0" 20:01:54 DEBUG }, 20:01:54 DEBUG "value": [ 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG [...] 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG "pod": "ds-idrepo-2" 20:01:54 DEBUG }, 20:01:54 DEBUG "value": [ 20:01:54 DEBUG 1713639714, 20:01:54 DEBUG "0" 20:01:54 DEBUG ] 20:01:54 DEBUG } 20:01:54 DEBUG ] 20:01:54 DEBUG } 20:01:54 DEBUG } 20:01:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639714" 20:01:54 INFO [http_cmd]: http status code OK 20:01:54 DEBUG --- status code --- 20:01:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:54 DEBUG --- http response --- 20:01:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:01:54 DEBUG ----- output ----- 20:01:54 DEBUG { 20:01:54 DEBUG "status": "success", 20:01:54 DEBUG "data": { 20:01:54 DEBUG "resultType": "vector", 20:01:54 DEBUG "result": [ 20:01:54 DEBUG { 20:01:54 DEBUG "metric": { 20:01:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:01:54 DEBUG }, 20:01:54 DEBUG "value": [ 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG [...] 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG 20:01:54 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:01:54 DEBUG }, 20:01:54 DEBUG "value": [ 20:01:54 DEBUG 1713639714, 20:01:54 DEBUG "0" 20:01:54 DEBUG ] 20:01:54 DEBUG } 20:01:54 DEBUG ] 20:01:54 DEBUG } 20:01:54 DEBUG } 20:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639715" 20:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639715" 20:01:55 INFO [http_cmd]: http status code OK 20:01:55 INFO [http_cmd]: http status code OK 20:01:55 DEBUG --- status code --- 20:01:55 DEBUG --- status code --- 20:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:55 DEBUG --- http response --- 20:01:55 DEBUG --- http response --- 20:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:55 DEBUG ----- output ----- 20:01:55 DEBUG { 20:01:55 DEBUG "status": "success", 20:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:55 DEBUG "data": { 20:01:55 DEBUG ----- output ----- 20:01:55 DEBUG "resultType": "vector", 20:01:55 DEBUG { 20:01:55 DEBUG "result": [ 20:01:55 DEBUG "status": "success", 20:01:55 DEBUG { 20:01:55 DEBUG "data": { 20:01:55 DEBUG "metric": { 20:01:55 DEBUG "resultType": "vector", 20:01:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:01:55 DEBUG "result": [ 20:01:55 DEBUG }, 20:01:55 DEBUG { 20:01:55 DEBUG "value": [ 20:01:55 DEBUG "metric": { 20:01:55 DEBUG 20:01:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:55 DEBUG 20:01:55 DEBUG }, 20:01:55 DEBUG 20:01:55 DEBUG [...] 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:01:55 DEBUG }, 20:01:55 DEBUG "value": [ 20:01:55 DEBUG 1713639715, 20:01:55 DEBUG "9.547652178162998" 20:01:55 DEBUG ] 20:01:55 DEBUG } 20:01:55 DEBUG ] 20:01:55 DEBUG "value": [ 20:01:55 DEBUG } 20:01:55 DEBUG 20:01:55 DEBUG } 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG [...] 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:55 DEBUG }, 20:01:55 DEBUG "value": [ 20:01:55 DEBUG 1713639715, 20:01:55 DEBUG "2834.7" 20:01:55 DEBUG ] 20:01:55 DEBUG } 20:01:55 DEBUG ] 20:01:55 DEBUG } 20:01:55 DEBUG } 20:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639715" 20:01:55 INFO [http_cmd]: http status code OK 20:01:55 DEBUG --- status code --- 20:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:55 DEBUG --- http response --- 20:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:55 DEBUG ----- output ----- 20:01:55 DEBUG { 20:01:55 DEBUG "status": "success", 20:01:55 DEBUG "data": { 20:01:55 DEBUG "resultType": "vector", 20:01:55 DEBUG "result": [ 20:01:55 DEBUG { 20:01:55 DEBUG "metric": { 20:01:55 DEBUG "instance": "10.142.0.107:9100" 20:01:55 DEBUG }, 20:01:55 DEBUG "value": [ 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG [...] 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG "instance": "10.142.0.43:9100" 20:01:55 DEBUG }, 20:01:55 DEBUG "value": [ 20:01:55 DEBUG 1713639715, 20:01:55 DEBUG "3650157005824" 20:01:55 DEBUG ] 20:01:55 DEBUG } 20:01:55 DEBUG ] 20:01:55 DEBUG } 20:01:55 DEBUG } 20:01:55 INFO 20:01:55 INFO [loop_until]: kubectl --namespace=xlou top pods 20:01:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:01:55 INFO [loop_until]: OK (rc = 0) 20:01:55 DEBUG --- stdout --- 20:01:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18307m 8654Mi am-767f6b67b5-qcqz2 18467m 9063Mi am-767f6b67b5-vbltz 19866m 8667Mi ds-cts-0 20952m 10787Mi ds-cts-1 18234m 10762Mi ds-cts-2 20689m 10921Mi ds-idrepo-0 8017m 3893Mi ds-idrepo-1 7940m 3882Mi ds-idrepo-2 8003m 3771Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 22m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9561m 797Mi 20:01:55 DEBUG --- stderr --- 20:01:55 DEBUG 20:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639715" 20:01:55 INFO [http_cmd]: http status code OK 20:01:55 DEBUG --- status code --- 20:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:55 DEBUG --- http response --- 20:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:55 DEBUG ----- output ----- 20:01:55 DEBUG { 20:01:55 DEBUG "status": "success", 20:01:55 DEBUG "data": { 20:01:55 DEBUG "resultType": "vector", 20:01:55 DEBUG "result": [ 20:01:55 DEBUG { 20:01:55 DEBUG "metric": { 20:01:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:55 DEBUG }, 20:01:55 DEBUG "value": [ 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG [...] 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:55 DEBUG }, 20:01:55 DEBUG "value": [ 20:01:55 DEBUG 1713639715, 20:01:55 DEBUG "2832.5333333333333" 20:01:55 DEBUG ] 20:01:55 DEBUG } 20:01:55 DEBUG ] 20:01:55 DEBUG } 20:01:55 DEBUG } 20:01:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639715" 20:01:55 INFO [http_cmd]: http status code OK 20:01:55 DEBUG --- status code --- 20:01:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:55 DEBUG --- http response --- 20:01:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:55 DEBUG ----- output ----- 20:01:55 DEBUG { 20:01:55 DEBUG "status": "success", 20:01:55 DEBUG "data": { 20:01:55 DEBUG "resultType": "vector", 20:01:55 DEBUG "result": [ 20:01:55 DEBUG { 20:01:55 DEBUG "metric": { 20:01:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:55 DEBUG }, 20:01:55 DEBUG "value": [ 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG [...] 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG 20:01:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:55 DEBUG }, 20:01:55 DEBUG "value": [ 20:01:55 DEBUG 1713639715, 20:01:55 DEBUG "2832.8333333333335" 20:01:55 DEBUG ] 20:01:55 DEBUG } 20:01:55 DEBUG ] 20:01:55 DEBUG } 20:01:55 DEBUG } 20:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639716" 20:01:56 INFO [http_cmd]: http status code OK 20:01:56 DEBUG --- status code --- 20:01:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:56 DEBUG --- http response --- 20:01:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:56 DEBUG ----- output ----- 20:01:56 DEBUG { 20:01:56 DEBUG "status": "success", 20:01:56 DEBUG "data": { 20:01:56 DEBUG "resultType": "vector", 20:01:56 DEBUG "result": [ 20:01:56 DEBUG { 20:01:56 DEBUG "metric": { 20:01:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:01:56 DEBUG }, 20:01:56 DEBUG "value": [ 20:01:56 DEBUG 20:01:56 DEBUG 20:01:56 DEBUG 20:01:56 DEBUG [...] 20:01:56 DEBUG 20:01:56 DEBUG 20:01:56 DEBUG 20:01:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:01:56 DEBUG }, 20:01:56 DEBUG "value": [ 20:01:56 DEBUG 1713639716, 20:01:56 DEBUG "9.503495312794852" 20:01:56 DEBUG ] 20:01:56 DEBUG } 20:01:56 DEBUG ] 20:01:56 DEBUG } 20:01:56 DEBUG } 20:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639716" 20:01:56 INFO [http_cmd]: http status code OK 20:01:56 DEBUG --- status code --- 20:01:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:56 DEBUG --- http response --- 20:01:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:56 DEBUG ----- output ----- 20:01:56 DEBUG { 20:01:56 DEBUG "status": "success", 20:01:56 DEBUG "data": { 20:01:56 DEBUG "resultType": "vector", 20:01:56 DEBUG "result": [ 20:01:56 DEBUG { 20:01:56 DEBUG "metric": { 20:01:56 DEBUG "pod": "ds-idrepo-0" 20:01:56 DEBUG }, 20:01:56 DEBUG "value": [ 20:01:56 DEBUG 20:01:56 DEBUG 20:01:56 DEBUG 20:01:56 DEBUG [...] 20:01:56 DEBUG 20:01:56 DEBUG 20:01:56 DEBUG 20:01:56 DEBUG "pod": "ds-idrepo-2" 20:01:56 DEBUG }, 20:01:56 DEBUG "value": [ 20:01:56 DEBUG 1713639716, 20:01:56 DEBUG "0" 20:01:56 DEBUG ] 20:01:56 DEBUG } 20:01:56 DEBUG ] 20:01:56 DEBUG } 20:01:56 DEBUG } 20:01:56 INFO 20:01:56 INFO [loop_until]: kubectl --namespace=xlou top node 20:01:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:01:56 INFO [loop_until]: OK (rc = 0) 20:01:56 DEBUG --- stdout --- 20:01:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6220Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8180m 27% 7644Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17631m 59% 11369Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3346Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20065m 67% 13980Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5183Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18519m 62% 11945Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15760m 52% 2964Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20766m 69% 10880Mi 9% gke-xlou-cdm-ds-0f601751-j488 8717m 29% 7162Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18239m 61% 15801Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8682m 29% 7845Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20650m 69% 13581Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9770m 61% 6397Mi 10% 20:01:56 DEBUG --- stderr --- 20:01:56 DEBUG 20:01:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639716" 20:01:56 INFO [http_cmd]: http status code OK 20:01:56 DEBUG --- status code --- 20:01:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:56 DEBUG --- http response --- 20:01:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:01:56 DEBUG ----- output ----- 20:01:56 DEBUG { 20:01:56 DEBUG "status": "success", 20:01:56 DEBUG "data": { 20:01:56 DEBUG "resultType": "vector", 20:01:56 DEBUG "result": [ 20:01:56 DEBUG { 20:01:56 DEBUG "metric": { 20:01:56 DEBUG "pod": "ds-cts-0" 20:01:56 DEBUG }, 20:01:56 DEBUG "value": [ 20:01:56 DEBUG 20:01:56 DEBUG 20:01:56 DEBUG 20:01:56 DEBUG [...] 20:01:56 DEBUG 20:01:56 DEBUG 20:01:56 DEBUG 20:01:56 DEBUG "pod": "ds-cts-2" 20:01:56 DEBUG }, 20:01:56 DEBUG "value": [ 20:01:56 DEBUG 1713639716, 20:01:56 DEBUG "0" 20:01:56 DEBUG ] 20:01:56 DEBUG } 20:01:56 DEBUG ] 20:01:56 DEBUG } 20:01:56 DEBUG } 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639717" 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 DEBUG --- status code --- 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG --- http response --- 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG { 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG "data": { 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG "result": [ 20:01:57 DEBUG { 20:01:57 DEBUG "metric": { 20:01:57 DEBUG "pod": "ds-idrepo-0" 20:01:57 DEBUG }, 20:01:57 DEBUG "value": [ 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG [...] 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG "pod": "ds-cts-2" 20:01:57 DEBUG }, 20:01:57 DEBUG "value": [ 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG "0" 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639717" 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 DEBUG --- status code --- 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG --- http response --- 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG { 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG "data": { 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG "result": [ 20:01:57 DEBUG { 20:01:57 DEBUG "metric": { 20:01:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:57 DEBUG }, 20:01:57 DEBUG "value": [ 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG [...] 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:57 DEBUG }, 20:01:57 DEBUG "value": [ 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG "510551.5" 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639717" 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639717" 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639717" 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639717" 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639717" 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639717" 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639717" 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639717" 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639717" 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639717" 20:01:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639717" 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 DEBUG --- status code --- 20:01:57 DEBUG --- status code --- 20:01:57 DEBUG --- status code --- 20:01:57 DEBUG --- status code --- 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 DEBUG --- status code --- 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG --- http response --- 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG --- http response --- 20:01:57 DEBUG --- status code --- 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG --- status code --- 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 DEBUG --- http response --- 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG --- http response --- 20:01:57 INFO [http_cmd]: http status code OK 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG --- status code --- 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG --- status code --- 20:01:57 DEBUG --- status code --- 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:57 DEBUG --- http response --- 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG --- http response --- 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:01:57 DEBUG --- status code --- 20:01:57 DEBUG --- http response --- 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG { 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:01:57 DEBUG { 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:57 DEBUG --- http response --- 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG --- http response --- 20:01:57 DEBUG --- http response --- 20:01:57 DEBUG { 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG { 20:01:57 DEBUG --- http response --- 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG "data": { 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG { 20:01:57 DEBUG "data": { 20:01:57 DEBUG { 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:01:57 DEBUG { 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG "data": { 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG "data": { 20:01:57 DEBUG "data": { 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG "result": [ 20:01:57 DEBUG { 20:01:57 DEBUG { 20:01:57 DEBUG { 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG "data": { 20:01:57 DEBUG "result": [ 20:01:57 DEBUG ----- output ----- 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG "data": { 20:01:57 DEBUG { 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG "result": [ 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG { 20:01:57 DEBUG { 20:01:57 DEBUG "result": [ 20:01:57 DEBUG "result": [ 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG "metric": { 20:01:57 DEBUG "data": { 20:01:57 DEBUG "data": { 20:01:57 DEBUG "data": { 20:01:57 DEBUG { 20:01:57 DEBUG "result": [ 20:01:57 DEBUG "metric": { 20:01:57 DEBUG "status": "success", 20:01:57 DEBUG { 20:01:57 DEBUG { 20:01:57 DEBUG "result": [ 20:01:57 DEBUG "instance": "10.142.0.107:9100" 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG "metric": { 20:01:57 DEBUG { 20:01:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:01:57 DEBUG "data": { 20:01:57 DEBUG "metric": { 20:01:57 DEBUG "metric": { 20:01:57 DEBUG { 20:01:57 DEBUG }, 20:01:57 DEBUG "result": [ 20:01:57 DEBUG "result": [ 20:01:57 DEBUG "result": [ 20:01:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:57 DEBUG "metric": { 20:01:57 DEBUG }, 20:01:57 DEBUG "resultType": "vector", 20:01:57 DEBUG "instance": "10.142.0.107:9100" 20:01:57 DEBUG "pod": "ds-cts-0" 20:01:57 DEBUG "metric": { 20:01:57 DEBUG "value": [ 20:01:57 DEBUG { 20:01:57 DEBUG { 20:01:57 DEBUG { 20:01:57 DEBUG }, 20:01:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:01:57 DEBUG "value": [ 20:01:57 DEBUG "result": [ 20:01:57 DEBUG }, 20:01:57 DEBUG }, 20:01:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:01:57 DEBUG 20:01:57 DEBUG "metric": { 20:01:57 DEBUG "metric": { 20:01:57 DEBUG "metric": { 20:01:57 DEBUG "value": [ 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG { 20:01:57 DEBUG "value": [ 20:01:57 DEBUG "value": [ 20:01:57 DEBUG }, 20:01:57 DEBUG 20:01:57 DEBUG "instance": "10.142.0.107:9100" 20:01:57 DEBUG "instance": "10.142.0.107:9100" 20:01:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:01:57 DEBUG 20:01:57 DEBUG }, 20:01:57 DEBUG 20:01:57 DEBUG "metric": { 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG "value": [ 20:01:57 DEBUG 20:01:57 DEBUG }, 20:01:57 DEBUG }, 20:01:57 DEBUG }, 20:01:57 DEBUG 20:01:57 DEBUG "value": [ 20:01:57 DEBUG 20:01:57 DEBUG "pod": "ds-cts-0" 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG [...] 20:01:57 DEBUG "value": [ 20:01:57 DEBUG "value": [ 20:01:57 DEBUG "value": [ 20:01:57 DEBUG [...] 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG }, 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG [...] 20:01:57 DEBUG 20:01:57 DEBUG "value": [ 20:01:57 DEBUG [...] 20:01:57 DEBUG [...] 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG [...] 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG [...] 20:01:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:57 DEBUG [...] 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG [...] 20:01:57 DEBUG [...] 20:01:57 DEBUG 20:01:57 DEBUG }, 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG "instance": "10.142.0.43:9100" 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG "value": [ 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG [...] 20:01:57 DEBUG "instance": "10.142.0.43:9100" 20:01:57 DEBUG "pod": "ds-cts-2" 20:01:57 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:01:57 DEBUG }, 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG }, 20:01:57 DEBUG }, 20:01:57 DEBUG }, 20:01:57 DEBUG "value": [ 20:01:57 DEBUG 20:01:57 DEBUG 20:01:57 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:01:57 DEBUG "0" 20:01:57 DEBUG }, 20:01:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:01:57 DEBUG 20:01:57 DEBUG "value": [ 20:01:57 DEBUG "value": [ 20:01:57 DEBUG "value": [ 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG "instance": "10.142.0.43:9100" 20:01:57 DEBUG "instance": "10.142.0.43:9100" 20:01:57 DEBUG }, 20:01:57 DEBUG ] 20:01:57 DEBUG "value": [ 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG "0" 20:01:57 DEBUG "0.05566666666667667" 20:01:57 DEBUG }, 20:01:57 DEBUG "value": [ 20:01:57 DEBUG } 20:01:57 DEBUG "0.157" 20:01:57 DEBUG }, 20:01:57 DEBUG "pod": "ds-cts-2" 20:01:57 DEBUG "0.16463333333332267" 20:01:57 DEBUG "0.025096806666666666" 20:01:57 DEBUG ] 20:01:57 DEBUG }, 20:01:57 DEBUG ] 20:01:57 DEBUG "value": [ 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG ] 20:01:57 DEBUG ] 20:01:57 DEBUG "value": [ 20:01:57 DEBUG }, 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG "value": [ 20:01:57 DEBUG } 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG "48893.13871386506" 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG "value": [ 20:01:57 DEBUG ] 20:01:57 DEBUG ] 20:01:57 DEBUG ] 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG ] 20:01:57 DEBUG "86302189.03333333" 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG ] 20:01:57 DEBUG "934518784" 20:01:57 DEBUG 1713639717, 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG "86305762.6" 20:01:57 DEBUG } 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG ] 20:01:57 DEBUG "28.633333333333333" 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG ] 20:01:57 DEBUG ] 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG } 20:01:57 DEBUG } 20:02:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639773" 20:02:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639773" 20:02:53 INFO [http_cmd]: http status code OK 20:02:53 ERROR [http_cmd]: ERROR 20:02:53 DEBUG --- status code --- 20:02:53 ERROR --- status code --- 20:02:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:02:53 DEBUG --- http response --- 20:02:53 ERROR --- http response --- 20:02:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:02:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:02:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:02:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639774" 20:02:54 INFO [http_cmd]: http status code OK 20:02:54 DEBUG --- status code --- 20:02:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:54 DEBUG --- http response --- 20:02:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:54 DEBUG ----- output ----- 20:02:54 DEBUG { 20:02:54 DEBUG "status": "success", 20:02:54 DEBUG "data": { 20:02:54 DEBUG "resultType": "vector", 20:02:54 DEBUG "result": [ 20:02:54 DEBUG { 20:02:54 DEBUG "metric": { 20:02:54 DEBUG "pod": "ds-idrepo-0" 20:02:54 DEBUG }, 20:02:54 DEBUG "value": [ 20:02:54 DEBUG 20:02:54 DEBUG 20:02:54 DEBUG 20:02:54 DEBUG [...] 20:02:54 DEBUG 20:02:54 DEBUG 20:02:54 DEBUG 20:02:54 DEBUG "pod": "ds-idrepo-2" 20:02:54 DEBUG }, 20:02:54 DEBUG "value": [ 20:02:54 DEBUG 1713639774, 20:02:54 DEBUG "0" 20:02:54 DEBUG ] 20:02:54 DEBUG } 20:02:54 DEBUG ] 20:02:54 DEBUG } 20:02:54 DEBUG } 20:02:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639774" 20:02:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639774" 20:02:54 INFO [http_cmd]: http status code OK 20:02:54 DEBUG --- status code --- 20:02:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:54 DEBUG --- http response --- 20:02:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:54 DEBUG ----- output ----- 20:02:54 DEBUG { 20:02:54 DEBUG "status": "success", 20:02:54 DEBUG "data": { 20:02:54 DEBUG "resultType": "vector", 20:02:54 DEBUG "result": [ 20:02:54 DEBUG { 20:02:54 DEBUG "metric": { 20:02:54 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:54 DEBUG }, 20:02:54 DEBUG "value": [ 20:02:54 DEBUG 20:02:54 DEBUG 20:02:54 DEBUG 20:02:54 DEBUG [...] 20:02:54 DEBUG 20:02:54 DEBUG 20:02:54 DEBUG 20:02:54 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:54 DEBUG }, 20:02:54 DEBUG "value": [ 20:02:54 DEBUG 1713639774, 20:02:54 DEBUG "2841.5666666666666" 20:02:54 DEBUG ] 20:02:54 DEBUG } 20:02:54 DEBUG ] 20:02:54 DEBUG } 20:02:54 DEBUG } 20:02:54 INFO [http_cmd]: http status code OK 20:02:54 DEBUG --- status code --- 20:02:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:54 DEBUG --- http response --- 20:02:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:54 DEBUG ----- output ----- 20:02:54 DEBUG { 20:02:54 DEBUG "status": "success", 20:02:54 DEBUG "data": { 20:02:54 DEBUG "resultType": "vector", 20:02:54 DEBUG "result": [ 20:02:54 DEBUG { 20:02:54 DEBUG "metric": { 20:02:54 DEBUG "pod": "ds-idrepo-0" 20:02:54 DEBUG }, 20:02:54 DEBUG "value": [ 20:02:54 DEBUG 20:02:54 DEBUG 20:02:54 DEBUG 20:02:54 DEBUG [...] 20:02:54 DEBUG 20:02:54 DEBUG 20:02:54 DEBUG 20:02:54 DEBUG "pod": "ds-idrepo-2" 20:02:54 DEBUG }, 20:02:54 DEBUG "value": [ 20:02:54 DEBUG 1713639774, 20:02:54 DEBUG "0" 20:02:54 DEBUG ] 20:02:54 DEBUG } 20:02:54 DEBUG ] 20:02:54 DEBUG } 20:02:54 DEBUG } 20:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639775" 20:02:55 INFO [http_cmd]: http status code OK 20:02:55 DEBUG --- status code --- 20:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:55 DEBUG --- http response --- 20:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:55 DEBUG ----- output ----- 20:02:55 DEBUG { 20:02:55 DEBUG "status": "success", 20:02:55 DEBUG "data": { 20:02:55 DEBUG "resultType": "vector", 20:02:55 DEBUG "result": [ 20:02:55 DEBUG { 20:02:55 DEBUG "metric": { 20:02:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:02:55 DEBUG }, 20:02:55 DEBUG "value": [ 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG [...] 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:02:55 DEBUG }, 20:02:55 DEBUG "value": [ 20:02:55 DEBUG 1713639775, 20:02:55 DEBUG "0" 20:02:55 DEBUG ] 20:02:55 DEBUG } 20:02:55 DEBUG ] 20:02:55 DEBUG } 20:02:55 DEBUG } 20:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639775" 20:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639775" 20:02:55 INFO [http_cmd]: http status code OK 20:02:55 DEBUG --- status code --- 20:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:55 DEBUG --- http response --- 20:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:55 DEBUG ----- output ----- 20:02:55 DEBUG { 20:02:55 DEBUG "status": "success", 20:02:55 DEBUG "data": { 20:02:55 DEBUG "resultType": "vector", 20:02:55 DEBUG "result": [ 20:02:55 DEBUG { 20:02:55 DEBUG "metric": { 20:02:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:55 DEBUG }, 20:02:55 DEBUG "value": [ 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG [...] 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:55 DEBUG }, 20:02:55 DEBUG "value": [ 20:02:55 DEBUG 1713639775, 20:02:55 DEBUG "2822.133333333333" 20:02:55 DEBUG ] 20:02:55 DEBUG } 20:02:55 DEBUG ] 20:02:55 DEBUG } 20:02:55 DEBUG } 20:02:55 INFO [http_cmd]: http status code OK 20:02:55 DEBUG --- status code --- 20:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:55 DEBUG --- http response --- 20:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:55 DEBUG ----- output ----- 20:02:55 DEBUG { 20:02:55 DEBUG "status": "success", 20:02:55 DEBUG "data": { 20:02:55 DEBUG "resultType": "vector", 20:02:55 DEBUG "result": [ 20:02:55 DEBUG { 20:02:55 DEBUG "metric": { 20:02:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:02:55 DEBUG }, 20:02:55 DEBUG "value": [ 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG [...] 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:02:55 DEBUG }, 20:02:55 DEBUG "value": [ 20:02:55 DEBUG 1713639775, 20:02:55 DEBUG "9.569311081196044" 20:02:55 DEBUG ] 20:02:55 DEBUG } 20:02:55 DEBUG ] 20:02:55 DEBUG } 20:02:55 DEBUG } 20:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639775" 20:02:55 INFO [http_cmd]: http status code OK 20:02:55 DEBUG --- status code --- 20:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:55 DEBUG --- http response --- 20:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:55 DEBUG ----- output ----- 20:02:55 DEBUG { 20:02:55 DEBUG "status": "success", 20:02:55 DEBUG "data": { 20:02:55 DEBUG "resultType": "vector", 20:02:55 DEBUG "result": [ 20:02:55 DEBUG { 20:02:55 DEBUG "metric": { 20:02:55 DEBUG "instance": "10.142.0.107:9100" 20:02:55 DEBUG }, 20:02:55 DEBUG "value": [ 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG [...] 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG "instance": "10.142.0.43:9100" 20:02:55 DEBUG }, 20:02:55 DEBUG "value": [ 20:02:55 DEBUG 1713639775, 20:02:55 DEBUG "3653861966848" 20:02:55 DEBUG ] 20:02:55 DEBUG } 20:02:55 DEBUG ] 20:02:55 DEBUG } 20:02:55 DEBUG } 20:02:55 INFO 20:02:55 INFO [loop_until]: kubectl --namespace=xlou top pods 20:02:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:02:55 INFO [loop_until]: OK (rc = 0) 20:02:55 DEBUG --- stdout --- 20:02:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18417m 8656Mi am-767f6b67b5-qcqz2 18416m 9063Mi am-767f6b67b5-vbltz 19845m 8668Mi ds-cts-0 20321m 10788Mi ds-cts-1 18290m 10765Mi ds-cts-2 19622m 10907Mi ds-idrepo-0 7769m 3893Mi ds-idrepo-1 7688m 3882Mi ds-idrepo-2 7630m 3771Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 16m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9228m 799Mi 20:02:55 DEBUG --- stderr --- 20:02:55 DEBUG 20:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639775" 20:02:55 INFO [http_cmd]: http status code OK 20:02:55 DEBUG --- status code --- 20:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:55 DEBUG --- http response --- 20:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:55 DEBUG ----- output ----- 20:02:55 DEBUG { 20:02:55 DEBUG "status": "success", 20:02:55 DEBUG "data": { 20:02:55 DEBUG "resultType": "vector", 20:02:55 DEBUG "result": [ 20:02:55 DEBUG { 20:02:55 DEBUG "metric": { 20:02:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:55 DEBUG }, 20:02:55 DEBUG "value": [ 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG [...] 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:55 DEBUG }, 20:02:55 DEBUG "value": [ 20:02:55 DEBUG 1713639775, 20:02:55 DEBUG "2823.3333333333335" 20:02:55 DEBUG ] 20:02:55 DEBUG } 20:02:55 DEBUG ] 20:02:55 DEBUG } 20:02:55 DEBUG } 20:02:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639775" 20:02:55 INFO [http_cmd]: http status code OK 20:02:55 DEBUG --- status code --- 20:02:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:55 DEBUG --- http response --- 20:02:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:55 DEBUG ----- output ----- 20:02:55 DEBUG { 20:02:55 DEBUG "status": "success", 20:02:55 DEBUG "data": { 20:02:55 DEBUG "resultType": "vector", 20:02:55 DEBUG "result": [ 20:02:55 DEBUG { 20:02:55 DEBUG "metric": { 20:02:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:55 DEBUG }, 20:02:55 DEBUG "value": [ 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG [...] 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG 20:02:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:55 DEBUG }, 20:02:55 DEBUG "value": [ 20:02:55 DEBUG 1713639775, 20:02:55 DEBUG "2820.5666666666666" 20:02:55 DEBUG ] 20:02:55 DEBUG } 20:02:55 DEBUG ] 20:02:55 DEBUG } 20:02:55 DEBUG } 20:02:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639776" 20:02:56 INFO [http_cmd]: http status code OK 20:02:56 DEBUG --- status code --- 20:02:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:56 DEBUG --- http response --- 20:02:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:02:56 DEBUG ----- output ----- 20:02:56 DEBUG { 20:02:56 DEBUG "status": "success", 20:02:56 DEBUG "data": { 20:02:56 DEBUG "resultType": "vector", 20:02:56 DEBUG "result": [ 20:02:56 DEBUG { 20:02:56 DEBUG "metric": { 20:02:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:02:56 DEBUG }, 20:02:56 DEBUG "value": [ 20:02:56 DEBUG 20:02:56 DEBUG 20:02:56 DEBUG 20:02:56 DEBUG [...] 20:02:56 DEBUG 20:02:56 DEBUG 20:02:56 DEBUG 20:02:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:02:56 DEBUG }, 20:02:56 DEBUG "value": [ 20:02:56 DEBUG 1713639776, 20:02:56 DEBUG "6.816820621653457" 20:02:56 DEBUG ] 20:02:56 DEBUG } 20:02:56 DEBUG ] 20:02:56 DEBUG } 20:02:56 DEBUG } 20:02:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639776" 20:02:56 INFO [http_cmd]: http status code OK 20:02:56 DEBUG --- status code --- 20:02:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:56 DEBUG --- http response --- 20:02:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:56 DEBUG ----- output ----- 20:02:56 DEBUG { 20:02:56 DEBUG "status": "success", 20:02:56 DEBUG "data": { 20:02:56 DEBUG "resultType": "vector", 20:02:56 DEBUG "result": [ 20:02:56 DEBUG { 20:02:56 DEBUG "metric": { 20:02:56 DEBUG "pod": "ds-idrepo-0" 20:02:56 DEBUG }, 20:02:56 DEBUG "value": [ 20:02:56 DEBUG 20:02:56 DEBUG 20:02:56 DEBUG 20:02:56 DEBUG [...] 20:02:56 DEBUG 20:02:56 DEBUG 20:02:56 DEBUG 20:02:56 DEBUG "pod": "ds-idrepo-2" 20:02:56 DEBUG }, 20:02:56 DEBUG "value": [ 20:02:56 DEBUG 1713639776, 20:02:56 DEBUG "0" 20:02:56 DEBUG ] 20:02:56 DEBUG } 20:02:56 DEBUG ] 20:02:56 DEBUG } 20:02:56 DEBUG } 20:02:56 INFO 20:02:56 INFO [loop_until]: kubectl --namespace=xlou top node 20:02:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:02:56 INFO [loop_until]: OK (rc = 0) 20:02:56 DEBUG --- stdout --- 20:02:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6205Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8194m 27% 7634Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18284m 61% 11374Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3339Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20086m 67% 13977Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 167m 0% 5182Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18432m 61% 11948Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15255m 51% 2961Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20811m 69% 10886Mi 9% gke-xlou-cdm-ds-0f601751-j488 8732m 29% 7153Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18239m 61% 15790Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8662m 29% 7834Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20106m 67% 13570Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9780m 61% 6398Mi 10% 20:02:56 DEBUG --- stderr --- 20:02:56 DEBUG 20:02:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639776" 20:02:56 INFO [http_cmd]: http status code OK 20:02:56 DEBUG --- status code --- 20:02:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:56 DEBUG --- http response --- 20:02:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:02:56 DEBUG ----- output ----- 20:02:56 DEBUG { 20:02:56 DEBUG "status": "success", 20:02:56 DEBUG "data": { 20:02:56 DEBUG "resultType": "vector", 20:02:56 DEBUG "result": [ 20:02:56 DEBUG { 20:02:56 DEBUG "metric": { 20:02:56 DEBUG "pod": "ds-cts-0" 20:02:56 DEBUG }, 20:02:56 DEBUG "value": [ 20:02:56 DEBUG 20:02:56 DEBUG 20:02:56 DEBUG 20:02:56 DEBUG [...] 20:02:56 DEBUG 20:02:56 DEBUG 20:02:56 DEBUG 20:02:56 DEBUG "pod": "ds-cts-2" 20:02:56 DEBUG }, 20:02:56 DEBUG "value": [ 20:02:56 DEBUG 1713639776, 20:02:56 DEBUG "0" 20:02:56 DEBUG ] 20:02:56 DEBUG } 20:02:56 DEBUG ] 20:02:56 DEBUG } 20:02:56 DEBUG } 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639777" 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 DEBUG --- status code --- 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG { 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG "data": { 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG "result": [ 20:02:57 DEBUG { 20:02:57 DEBUG "metric": { 20:02:57 DEBUG "pod": "ds-idrepo-0" 20:02:57 DEBUG }, 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG [...] 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG "pod": "ds-cts-2" 20:02:57 DEBUG }, 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG "0" 20:02:57 DEBUG ] 20:02:57 DEBUG } 20:02:57 DEBUG ] 20:02:57 DEBUG } 20:02:57 DEBUG } 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639777" 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 DEBUG --- status code --- 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG { 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG "data": { 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG "result": [ 20:02:57 DEBUG { 20:02:57 DEBUG "metric": { 20:02:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:57 DEBUG }, 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG [...] 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:57 DEBUG }, 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG "510487.5" 20:02:57 DEBUG ] 20:02:57 DEBUG } 20:02:57 DEBUG ] 20:02:57 DEBUG } 20:02:57 DEBUG } 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639777" 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639777" 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639777" 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639777" 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639777" 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639777" 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639777" 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639777" 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639777" 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639777" 20:02:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639777" 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 DEBUG --- status code --- 20:02:57 DEBUG --- status code --- 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 DEBUG --- status code --- 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 DEBUG --- status code --- 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 DEBUG --- status code --- 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 DEBUG --- status code --- 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 DEBUG --- status code --- 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG --- status code --- 20:02:57 DEBUG --- status code --- 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG --- status code --- 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG { 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 DEBUG { 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:02:57 DEBUG { 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG { 20:02:57 DEBUG { 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG "data": { 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:57 DEBUG "data": { 20:02:57 DEBUG { 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG { 20:02:57 DEBUG "data": { 20:02:57 DEBUG { 20:02:57 DEBUG { 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG "data": { 20:02:57 INFO [http_cmd]: http status code OK 20:02:57 DEBUG "data": { 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG "result": [ 20:02:57 DEBUG { 20:02:57 DEBUG "result": [ 20:02:57 DEBUG "data": { 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG --- status code --- 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG "data": { 20:02:57 DEBUG "result": [ 20:02:57 DEBUG "data": { 20:02:57 DEBUG "data": { 20:02:57 DEBUG { 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG { 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG "result": [ 20:02:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:02:57 DEBUG "result": [ 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG { 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG "metric": { 20:02:57 DEBUG "data": { 20:02:57 DEBUG "metric": { 20:02:57 DEBUG "result": [ 20:02:57 DEBUG { 20:02:57 DEBUG --- http response --- 20:02:57 DEBUG { 20:02:57 DEBUG "result": [ 20:02:57 DEBUG "metric": { 20:02:57 DEBUG "result": [ 20:02:57 DEBUG "result": [ 20:02:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG "instance": "10.142.0.107:9100" 20:02:57 DEBUG { 20:02:57 DEBUG "metric": { 20:02:57 DEBUG "metric": { 20:02:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:02:57 DEBUG { 20:02:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:57 DEBUG { 20:02:57 DEBUG { 20:02:57 DEBUG }, 20:02:57 DEBUG "result": [ 20:02:57 DEBUG }, 20:02:57 DEBUG "metric": { 20:02:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:02:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:02:57 DEBUG ----- output ----- 20:02:57 DEBUG "metric": { 20:02:57 DEBUG }, 20:02:57 DEBUG "metric": { 20:02:57 DEBUG "metric": { 20:02:57 DEBUG "value": [ 20:02:57 DEBUG { 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 20:02:57 DEBUG }, 20:02:57 DEBUG }, 20:02:57 DEBUG { 20:02:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:02:57 DEBUG "value": [ 20:02:57 DEBUG "pod": "ds-cts-0" 20:02:57 DEBUG "pod": "ds-cts-0" 20:02:57 DEBUG 20:02:57 DEBUG "metric": { 20:02:57 DEBUG 20:02:57 DEBUG "instance": "10.142.0.107:9100" 20:02:57 DEBUG "value": [ 20:02:57 DEBUG "value": [ 20:02:57 DEBUG "status": "success", 20:02:57 DEBUG }, 20:02:57 DEBUG 20:02:57 DEBUG }, 20:02:57 DEBUG }, 20:02:57 DEBUG 20:02:57 DEBUG "instance": "10.142.0.107:9100" 20:02:57 DEBUG 20:02:57 DEBUG }, 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG "data": { 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 20:02:57 DEBUG "value": [ 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 20:02:57 DEBUG }, 20:02:57 DEBUG [...] 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG "resultType": "vector", 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG [...] 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG [...] 20:02:57 DEBUG 20:02:57 DEBUG [...] 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG [...] 20:02:57 DEBUG "result": [ 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG [...] 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG [...] 20:02:57 DEBUG { 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG [...] 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG "instance": "10.142.0.43:9100" 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG "metric": { 20:02:57 DEBUG "instance": "10.142.0.107:9100" 20:02:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:57 DEBUG }, 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG [...] 20:02:57 DEBUG 20:02:57 DEBUG }, 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG }, 20:02:57 DEBUG 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 20:02:57 DEBUG 20:02:57 DEBUG "value": [ 20:02:57 DEBUG [...] 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:02:57 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:02:57 DEBUG "pod": "ds-cts-2" 20:02:57 DEBUG 20:02:57 DEBUG }, 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG "0.163333333333361" 20:02:57 DEBUG ] 20:02:57 DEBUG "instance": "10.142.0.43:9100" 20:02:57 DEBUG 20:02:57 DEBUG "0.157" 20:02:57 DEBUG ] 20:02:57 DEBUG }, 20:02:57 DEBUG }, 20:02:57 DEBUG 20:02:57 DEBUG "value": [ 20:02:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:02:57 DEBUG } 20:02:57 DEBUG 20:02:57 DEBUG }, 20:02:57 DEBUG 20:02:57 DEBUG } 20:02:57 DEBUG }, 20:02:57 DEBUG "value": [ 20:02:57 DEBUG "value": [ 20:02:57 DEBUG "instance": "10.142.0.43:9100" 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG }, 20:02:57 DEBUG ] 20:02:57 DEBUG 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 20:02:57 DEBUG ] 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG }, 20:02:57 DEBUG "0" 20:02:57 DEBUG ] 20:02:57 DEBUG "value": [ 20:02:57 DEBUG 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG "pod": "ds-cts-2" 20:02:57 DEBUG } 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG "27075.221573404484" 20:02:57 DEBUG "0.017933333333333336" 20:02:57 DEBUG "value": [ 20:02:57 DEBUG } 20:02:57 DEBUG } 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG [...] 20:02:57 DEBUG "0.058666666666666374" 20:02:57 DEBUG }, 20:02:57 DEBUG } 20:02:57 DEBUG "0" 20:02:57 DEBUG ] 20:02:57 DEBUG ] 20:02:57 DEBUG } 20:02:57 DEBUG } 20:02:57 DEBUG ] 20:02:57 DEBUG "937418752" 20:02:57 DEBUG 20:02:57 DEBUG ] 20:02:57 DEBUG "value": [ 20:02:57 DEBUG ] 20:02:57 DEBUG } 20:02:57 DEBUG ] 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG } 20:02:57 DEBUG ] 20:02:57 DEBUG 20:02:57 DEBUG } 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG } 20:02:57 DEBUG ] 20:02:57 DEBUG } 20:02:57 DEBUG "85549602.73333333" 20:02:57 DEBUG } 20:02:57 DEBUG } 20:02:57 DEBUG 20:02:57 DEBUG ] 20:02:57 DEBUG "29.433333333333334" 20:02:57 DEBUG } 20:02:57 DEBUG ] 20:02:57 DEBUG } 20:02:57 DEBUG ] 20:02:57 DEBUG ] 20:02:57 DEBUG "instance": "10.142.0.43:9100" 20:02:57 DEBUG } 20:02:57 DEBUG ] 20:02:57 DEBUG } 20:02:57 DEBUG } 20:02:57 DEBUG } 20:02:57 DEBUG ] 20:02:57 DEBUG }, 20:02:57 DEBUG } 20:02:57 DEBUG } 20:02:57 DEBUG } 20:02:57 DEBUG } 20:02:57 DEBUG } 20:02:57 DEBUG "value": [ 20:02:57 DEBUG ] 20:02:57 DEBUG } 20:02:57 DEBUG } 20:02:57 DEBUG 1713639777, 20:02:57 DEBUG } 20:02:57 DEBUG "85554447.66666667" 20:02:57 DEBUG } 20:02:57 DEBUG ] 20:02:57 DEBUG } 20:02:57 DEBUG ] 20:02:57 DEBUG } 20:02:57 DEBUG } 20:03:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639833" 20:03:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639833" 20:03:53 ERROR [http_cmd]: ERROR 20:03:53 INFO [http_cmd]: http status code OK 20:03:53 ERROR --- status code --- 20:03:53 DEBUG --- status code --- 20:03:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:03:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:53 ERROR --- http response --- 20:03:53 DEBUG --- http response --- 20:03:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:03:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:03:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:03:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639834" 20:03:54 INFO [http_cmd]: http status code OK 20:03:54 DEBUG --- status code --- 20:03:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:54 DEBUG --- http response --- 20:03:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:54 DEBUG ----- output ----- 20:03:54 DEBUG { 20:03:54 DEBUG "status": "success", 20:03:54 DEBUG "data": { 20:03:54 DEBUG "resultType": "vector", 20:03:54 DEBUG "result": [ 20:03:54 DEBUG { 20:03:54 DEBUG "metric": { 20:03:54 DEBUG "pod": "ds-idrepo-0" 20:03:54 DEBUG }, 20:03:54 DEBUG "value": [ 20:03:54 DEBUG 20:03:54 DEBUG 20:03:54 DEBUG 20:03:54 DEBUG [...] 20:03:54 DEBUG 20:03:54 DEBUG 20:03:54 DEBUG 20:03:54 DEBUG "pod": "ds-idrepo-2" 20:03:54 DEBUG }, 20:03:54 DEBUG "value": [ 20:03:54 DEBUG 1713639834, 20:03:54 DEBUG "0" 20:03:54 DEBUG ] 20:03:54 DEBUG } 20:03:54 DEBUG ] 20:03:54 DEBUG } 20:03:54 DEBUG } 20:03:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639834" 20:03:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639834" 20:03:54 INFO [http_cmd]: http status code OK 20:03:54 DEBUG --- status code --- 20:03:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:54 INFO [http_cmd]: http status code OK 20:03:54 DEBUG --- http response --- 20:03:54 DEBUG --- status code --- 20:03:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:54 DEBUG ----- output ----- 20:03:54 DEBUG --- http response --- 20:03:54 DEBUG { 20:03:54 DEBUG "status": "success", 20:03:54 DEBUG "data": { 20:03:54 DEBUG "resultType": "vector", 20:03:54 DEBUG "result": [ 20:03:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:54 DEBUG { 20:03:54 DEBUG ----- output ----- 20:03:54 DEBUG "metric": { 20:03:54 DEBUG { 20:03:54 DEBUG "pod": "ds-idrepo-0" 20:03:54 DEBUG "status": "success", 20:03:54 DEBUG }, 20:03:54 DEBUG "data": { 20:03:54 DEBUG "value": [ 20:03:54 DEBUG "resultType": "vector", 20:03:54 DEBUG "result": [ 20:03:54 DEBUG 20:03:54 DEBUG { 20:03:54 DEBUG 20:03:54 DEBUG "metric": { 20:03:54 DEBUG 20:03:54 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:54 DEBUG [...] 20:03:54 DEBUG }, 20:03:54 DEBUG 20:03:54 DEBUG "value": [ 20:03:54 DEBUG 20:03:54 DEBUG 20:03:54 DEBUG 20:03:54 DEBUG 20:03:54 DEBUG "pod": "ds-idrepo-2" 20:03:54 DEBUG 20:03:54 DEBUG }, 20:03:54 DEBUG [...] 20:03:54 DEBUG "value": [ 20:03:54 DEBUG 20:03:54 DEBUG 1713639834, 20:03:54 DEBUG 20:03:54 DEBUG "0" 20:03:54 DEBUG 20:03:54 DEBUG ] 20:03:54 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:54 DEBUG } 20:03:54 DEBUG }, 20:03:54 DEBUG ] 20:03:54 DEBUG "value": [ 20:03:54 DEBUG } 20:03:54 DEBUG 1713639834, 20:03:54 DEBUG } 20:03:54 DEBUG "2841.3" 20:03:54 DEBUG ] 20:03:54 DEBUG } 20:03:54 DEBUG ] 20:03:54 DEBUG } 20:03:54 DEBUG } 20:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639835" 20:03:55 INFO [http_cmd]: http status code OK 20:03:55 DEBUG --- status code --- 20:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:55 DEBUG --- http response --- 20:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 20:03:55 DEBUG ----- output ----- 20:03:55 DEBUG { 20:03:55 DEBUG "status": "success", 20:03:55 DEBUG "data": { 20:03:55 DEBUG "resultType": "vector", 20:03:55 DEBUG "result": [ 20:03:55 DEBUG { 20:03:55 DEBUG "metric": { 20:03:55 DEBUG "pod": "ds-idrepo-1" 20:03:55 DEBUG }, 20:03:55 DEBUG "value": [ 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG [...] 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:03:55 DEBUG }, 20:03:55 DEBUG "value": [ 20:03:55 DEBUG 1713639835, 20:03:55 DEBUG "0" 20:03:55 DEBUG ] 20:03:55 DEBUG } 20:03:55 DEBUG ] 20:03:55 DEBUG } 20:03:55 DEBUG } 20:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639835" 20:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639835" 20:03:55 INFO [http_cmd]: http status code OK 20:03:55 DEBUG --- status code --- 20:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:55 DEBUG --- http response --- 20:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:55 DEBUG ----- output ----- 20:03:55 DEBUG { 20:03:55 DEBUG "status": "success", 20:03:55 DEBUG "data": { 20:03:55 DEBUG "resultType": "vector", 20:03:55 DEBUG "result": [ 20:03:55 DEBUG { 20:03:55 DEBUG "metric": { 20:03:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:55 DEBUG }, 20:03:55 DEBUG "value": [ 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG [...] 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 INFO [http_cmd]: http status code OK 20:03:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:55 DEBUG --- status code --- 20:03:55 DEBUG }, 20:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:55 DEBUG "value": [ 20:03:55 DEBUG 1713639835, 20:03:55 DEBUG "2832.6" 20:03:55 DEBUG ] 20:03:55 DEBUG } 20:03:55 DEBUG ] 20:03:55 DEBUG } 20:03:55 DEBUG } 20:03:55 DEBUG --- http response --- 20:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:55 DEBUG ----- output ----- 20:03:55 DEBUG { 20:03:55 DEBUG "status": "success", 20:03:55 DEBUG "data": { 20:03:55 DEBUG "resultType": "vector", 20:03:55 DEBUG "result": [ 20:03:55 DEBUG { 20:03:55 DEBUG "metric": { 20:03:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:03:55 DEBUG }, 20:03:55 DEBUG "value": [ 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG [...] 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:03:55 DEBUG }, 20:03:55 DEBUG "value": [ 20:03:55 DEBUG 1713639835, 20:03:55 DEBUG "9.513217053921002" 20:03:55 DEBUG ] 20:03:55 DEBUG } 20:03:55 DEBUG ] 20:03:55 DEBUG } 20:03:55 DEBUG } 20:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639835" 20:03:55 INFO [http_cmd]: http status code OK 20:03:55 DEBUG --- status code --- 20:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:55 DEBUG --- http response --- 20:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:55 DEBUG ----- output ----- 20:03:55 DEBUG { 20:03:55 DEBUG "status": "success", 20:03:55 DEBUG "data": { 20:03:55 DEBUG "resultType": "vector", 20:03:55 DEBUG "result": [ 20:03:55 DEBUG { 20:03:55 DEBUG "metric": { 20:03:55 DEBUG "instance": "10.142.0.107:9100" 20:03:55 DEBUG }, 20:03:55 DEBUG "value": [ 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG [...] 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG "instance": "10.142.0.43:9100" 20:03:55 DEBUG }, 20:03:55 DEBUG "value": [ 20:03:55 DEBUG 1713639835, 20:03:55 DEBUG "3657564328960" 20:03:55 DEBUG ] 20:03:55 DEBUG } 20:03:55 DEBUG ] 20:03:55 DEBUG } 20:03:55 DEBUG } 20:03:55 INFO 20:03:55 INFO [loop_until]: kubectl --namespace=xlou top pods 20:03:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:03:55 INFO [loop_until]: OK (rc = 0) 20:03:55 DEBUG --- stdout --- 20:03:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18575m 8657Mi am-767f6b67b5-qcqz2 18387m 9065Mi am-767f6b67b5-vbltz 20506m 8669Mi ds-cts-0 20018m 10788Mi ds-cts-1 17998m 10759Mi ds-cts-2 20423m 10948Mi ds-idrepo-0 8012m 3893Mi ds-idrepo-1 7864m 3883Mi ds-idrepo-2 7805m 3771Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 14m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9602m 801Mi 20:03:55 DEBUG --- stderr --- 20:03:55 DEBUG 20:03:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639835" 20:03:55 INFO [http_cmd]: http status code OK 20:03:55 DEBUG --- status code --- 20:03:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:55 DEBUG --- http response --- 20:03:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:55 DEBUG ----- output ----- 20:03:55 DEBUG { 20:03:55 DEBUG "status": "success", 20:03:55 DEBUG "data": { 20:03:55 DEBUG "resultType": "vector", 20:03:55 DEBUG "result": [ 20:03:55 DEBUG { 20:03:55 DEBUG "metric": { 20:03:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:55 DEBUG }, 20:03:55 DEBUG "value": [ 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG [...] 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG 20:03:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:55 DEBUG }, 20:03:55 DEBUG "value": [ 20:03:55 DEBUG 1713639835, 20:03:55 DEBUG "2833.8" 20:03:55 DEBUG ] 20:03:55 DEBUG } 20:03:55 DEBUG ] 20:03:55 DEBUG } 20:03:55 DEBUG } 20:03:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639836" 20:03:56 INFO [http_cmd]: http status code OK 20:03:56 DEBUG --- status code --- 20:03:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:56 DEBUG --- http response --- 20:03:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:56 DEBUG ----- output ----- 20:03:56 DEBUG { 20:03:56 DEBUG "status": "success", 20:03:56 DEBUG "data": { 20:03:56 DEBUG "resultType": "vector", 20:03:56 DEBUG "result": [ 20:03:56 DEBUG { 20:03:56 DEBUG "metric": { 20:03:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:56 DEBUG }, 20:03:56 DEBUG "value": [ 20:03:56 DEBUG 20:03:56 DEBUG 20:03:56 DEBUG 20:03:56 DEBUG [...] 20:03:56 DEBUG 20:03:56 DEBUG 20:03:56 DEBUG 20:03:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:56 DEBUG }, 20:03:56 DEBUG "value": [ 20:03:56 DEBUG 1713639836, 20:03:56 DEBUG "2834.3" 20:03:56 DEBUG ] 20:03:56 DEBUG } 20:03:56 DEBUG ] 20:03:56 DEBUG } 20:03:56 DEBUG } 20:03:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639836" 20:03:56 INFO [http_cmd]: http status code OK 20:03:56 DEBUG --- status code --- 20:03:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:56 DEBUG --- http response --- 20:03:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:56 DEBUG ----- output ----- 20:03:56 DEBUG { 20:03:56 DEBUG "status": "success", 20:03:56 DEBUG "data": { 20:03:56 DEBUG "resultType": "vector", 20:03:56 DEBUG "result": [ 20:03:56 DEBUG { 20:03:56 DEBUG "metric": { 20:03:56 DEBUG "pod": "ds-idrepo-1" 20:03:56 DEBUG }, 20:03:56 DEBUG "value": [ 20:03:56 DEBUG 20:03:56 DEBUG 20:03:56 DEBUG 20:03:56 DEBUG [...] 20:03:56 DEBUG 20:03:56 DEBUG 20:03:56 DEBUG 20:03:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:03:56 DEBUG }, 20:03:56 DEBUG "value": [ 20:03:56 DEBUG 1713639836, 20:03:56 DEBUG "0" 20:03:56 DEBUG ] 20:03:56 DEBUG } 20:03:56 DEBUG ] 20:03:56 DEBUG } 20:03:56 DEBUG } 20:03:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639836" 20:03:56 INFO [http_cmd]: http status code OK 20:03:56 DEBUG --- status code --- 20:03:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:56 DEBUG --- http response --- 20:03:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:56 DEBUG ----- output ----- 20:03:56 DEBUG { 20:03:56 DEBUG "status": "success", 20:03:56 DEBUG "data": { 20:03:56 DEBUG "resultType": "vector", 20:03:56 DEBUG "result": [ 20:03:56 DEBUG { 20:03:56 DEBUG "metric": { 20:03:56 DEBUG "pod": "ds-idrepo-0" 20:03:56 DEBUG }, 20:03:56 DEBUG "value": [ 20:03:56 DEBUG 20:03:56 DEBUG 20:03:56 DEBUG 20:03:56 DEBUG [...] 20:03:56 DEBUG 20:03:56 DEBUG 20:03:56 DEBUG 20:03:56 DEBUG "pod": "ds-idrepo-2" 20:03:56 DEBUG }, 20:03:56 DEBUG "value": [ 20:03:56 DEBUG 1713639836, 20:03:56 DEBUG "0" 20:03:56 DEBUG ] 20:03:56 DEBUG } 20:03:56 DEBUG ] 20:03:56 DEBUG } 20:03:56 DEBUG } 20:03:56 INFO 20:03:56 INFO [loop_until]: kubectl --namespace=xlou top node 20:03:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:03:56 INFO [loop_until]: OK (rc = 0) 20:03:56 DEBUG --- stdout --- 20:03:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6213Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8163m 27% 7643Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18113m 60% 11377Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3340Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20117m 67% 13988Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5180Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18454m 61% 11956Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15683m 52% 2969Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20845m 69% 10899Mi 9% gke-xlou-cdm-ds-0f601751-j488 8729m 29% 7176Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18137m 60% 15788Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8623m 28% 7854Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20577m 68% 13618Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9448m 59% 6394Mi 10% 20:03:56 DEBUG --- stderr --- 20:03:56 DEBUG 20:03:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639836" 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 DEBUG --- status code --- 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG { 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG "data": { 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG "result": [ 20:03:57 DEBUG { 20:03:57 DEBUG "metric": { 20:03:57 DEBUG "pod": "ds-cts-0" 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG [...] 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG "pod": "ds-cts-2" 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 1713639836, 20:03:57 DEBUG "0" 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639837" 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 DEBUG --- status code --- 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG { 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG "data": { 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG "result": [ 20:03:57 DEBUG { 20:03:57 DEBUG "metric": { 20:03:57 DEBUG "pod": "ds-idrepo-0" 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG [...] 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG "pod": "ds-cts-2" 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG "0" 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639837" 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 DEBUG --- status code --- 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG { 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG "data": { 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG "result": [ 20:03:57 DEBUG { 20:03:57 DEBUG "metric": { 20:03:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG [...] 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG "510139" 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639837" 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 DEBUG --- status code --- 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG { 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG "data": { 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG "result": [ 20:03:57 DEBUG { 20:03:57 DEBUG "metric": { 20:03:57 DEBUG "instance": "10.142.0.107:9100" 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG [...] 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG "instance": "10.142.0.43:9100" 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG "0.23620000000000232" 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639837" 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639837" 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639837" 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639837" 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639837" 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639837" 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639837" 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639837" 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639837" 20:03:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639837" 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 DEBUG --- status code --- 20:03:57 DEBUG --- status code --- 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 DEBUG --- status code --- 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 DEBUG --- status code --- 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG --- status code --- 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG --- status code --- 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG --- http response --- 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG --- status code --- 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG --- status code --- 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG --- status code --- 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 INFO [http_cmd]: http status code OK 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG { 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG --- status code --- 20:03:57 DEBUG { 20:03:57 DEBUG { 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG { 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG "data": { 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG { 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG --- http response --- 20:03:57 DEBUG "data": { 20:03:57 DEBUG "data": { 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG { 20:03:57 DEBUG { 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG "data": { 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG { 20:03:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG "result": [ 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG "data": { 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG { 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG ----- output ----- 20:03:57 DEBUG "result": [ 20:03:57 DEBUG { 20:03:57 DEBUG { 20:03:57 DEBUG "data": { 20:03:57 DEBUG "data": { 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG "result": [ 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG "data": { 20:03:57 DEBUG { 20:03:57 DEBUG "result": [ 20:03:57 DEBUG "metric": { 20:03:57 DEBUG "metric": { 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG "result": [ 20:03:57 DEBUG { 20:03:57 DEBUG "data": { 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG "status": "success", 20:03:57 DEBUG { 20:03:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:57 DEBUG "pod": "ds-cts-0" 20:03:57 DEBUG }, 20:03:57 DEBUG "result": [ 20:03:57 DEBUG { 20:03:57 DEBUG "metric": { 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG "result": [ 20:03:57 DEBUG "data": { 20:03:57 DEBUG "metric": { 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG "result": [ 20:03:57 DEBUG { 20:03:57 DEBUG "metric": { 20:03:57 DEBUG "pod": "ds-idrepo-1" 20:03:57 DEBUG "result": [ 20:03:57 DEBUG { 20:03:57 DEBUG "resultType": "vector", 20:03:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG { 20:03:57 DEBUG "metric": { 20:03:57 DEBUG "pod": "ds-idrepo-1" 20:03:57 DEBUG }, 20:03:57 DEBUG { 20:03:57 DEBUG "metric": { 20:03:57 DEBUG "result": [ 20:03:57 DEBUG }, 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG "metric": { 20:03:57 DEBUG "instance": "10.142.0.107:9100" 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG "metric": { 20:03:57 DEBUG "instance": "10.142.0.107:9100" 20:03:57 DEBUG { 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG "instance": "10.142.0.107:9100" 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG "pod": "ds-cts-0" 20:03:57 DEBUG }, 20:03:57 DEBUG "metric": { 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG [...] 20:03:57 DEBUG 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG }, 20:03:57 DEBUG [...] 20:03:57 DEBUG [...] 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG "pod": "ds-cts-2" 20:03:57 DEBUG }, 20:03:57 DEBUG [...] 20:03:57 DEBUG [...] 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG [...] 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG [...] 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:57 DEBUG 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG [...] 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG }, 20:03:57 DEBUG 20:03:57 DEBUG "0.0008000000000000007" 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG [...] 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG [...] 20:03:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG ] 20:03:57 DEBUG "instance": "10.142.0.43:9100" 20:03:57 DEBUG "pod": "idm-b85f9d978-txpwv" 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG }, 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG "pod": "idm-b85f9d978-txpwv" 20:03:57 DEBUG } 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 20:03:57 DEBUG 20:03:57 DEBUG "instance": "10.142.0.43:9100" 20:03:57 DEBUG 20:03:57 DEBUG "value": [ 20:03:57 DEBUG "0" 20:03:57 DEBUG }, 20:03:57 DEBUG ] 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG }, 20:03:57 DEBUG 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG ] 20:03:57 DEBUG "value": [ 20:03:57 DEBUG } 20:03:57 DEBUG "85558373.13333333" 20:03:57 DEBUG 20:03:57 DEBUG "pod": "ds-cts-2" 20:03:57 DEBUG "0" 20:03:57 DEBUG ] 20:03:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:03:57 DEBUG "0.157" 20:03:57 DEBUG } 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG "instance": "10.142.0.43:9100" 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG } 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG ] 20:03:57 DEBUG "45446545.34798534" 20:03:57 DEBUG } 20:03:57 DEBUG }, 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG ] 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG ] 20:03:57 DEBUG ] 20:03:57 DEBUG "value": [ 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG "23.866666666666667" 20:03:57 DEBUG } 20:03:57 DEBUG "937259008" 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG 1713639837, 20:03:57 DEBUG ] 20:03:57 DEBUG "85562172.03333333" 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 DEBUG "0.07733333333333121" 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 DEBUG ] 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 DEBUG } 20:03:57 DEBUG } 20:04:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639893" 20:04:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639893" 20:04:53 ERROR [http_cmd]: ERROR 20:04:53 INFO [http_cmd]: http status code OK 20:04:53 ERROR --- status code --- 20:04:53 DEBUG --- status code --- 20:04:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:04:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:53 ERROR --- http response --- 20:04:53 DEBUG --- http response --- 20:04:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:04:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:04:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:04:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639894" 20:04:54 INFO [http_cmd]: http status code OK 20:04:54 DEBUG --- status code --- 20:04:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:54 DEBUG --- http response --- 20:04:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:54 DEBUG ----- output ----- 20:04:54 DEBUG { 20:04:54 DEBUG "status": "success", 20:04:54 DEBUG "data": { 20:04:54 DEBUG "resultType": "vector", 20:04:54 DEBUG "result": [ 20:04:54 DEBUG { 20:04:54 DEBUG "metric": { 20:04:54 DEBUG "pod": "ds-idrepo-0" 20:04:54 DEBUG }, 20:04:54 DEBUG "value": [ 20:04:54 DEBUG 20:04:54 DEBUG 20:04:54 DEBUG 20:04:54 DEBUG [...] 20:04:54 DEBUG 20:04:54 DEBUG 20:04:54 DEBUG 20:04:54 DEBUG "pod": "ds-idrepo-2" 20:04:54 DEBUG }, 20:04:54 DEBUG "value": [ 20:04:54 DEBUG 1713639894, 20:04:54 DEBUG "0" 20:04:54 DEBUG ] 20:04:54 DEBUG } 20:04:54 DEBUG ] 20:04:54 DEBUG } 20:04:54 DEBUG } 20:04:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639894" 20:04:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639894" 20:04:54 INFO [http_cmd]: http status code OK 20:04:54 DEBUG --- status code --- 20:04:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:54 DEBUG --- http response --- 20:04:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:54 DEBUG ----- output ----- 20:04:54 DEBUG { 20:04:54 INFO [http_cmd]: http status code OK 20:04:54 DEBUG "status": "success", 20:04:54 DEBUG --- status code --- 20:04:54 DEBUG "data": { 20:04:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:54 DEBUG "resultType": "vector", 20:04:54 DEBUG --- http response --- 20:04:54 DEBUG "result": [ 20:04:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:54 DEBUG { 20:04:54 DEBUG ----- output ----- 20:04:54 DEBUG "metric": { 20:04:54 DEBUG { 20:04:54 DEBUG "pod": "ds-idrepo-0" 20:04:54 DEBUG }, 20:04:54 DEBUG "value": [ 20:04:54 DEBUG 20:04:54 DEBUG 20:04:54 DEBUG "status": "success", 20:04:54 DEBUG 20:04:54 DEBUG "data": { 20:04:54 DEBUG [...] 20:04:54 DEBUG "resultType": "vector", 20:04:54 DEBUG 20:04:54 DEBUG "result": [ 20:04:54 DEBUG 20:04:54 DEBUG { 20:04:54 DEBUG 20:04:54 DEBUG "metric": { 20:04:54 DEBUG "pod": "ds-idrepo-2" 20:04:54 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:54 DEBUG }, 20:04:54 DEBUG }, 20:04:54 DEBUG "value": [ 20:04:54 DEBUG "value": [ 20:04:54 DEBUG 1713639894, 20:04:54 DEBUG 20:04:54 DEBUG "0" 20:04:54 DEBUG 20:04:54 DEBUG ] 20:04:54 DEBUG 20:04:54 DEBUG } 20:04:54 DEBUG [...] 20:04:54 DEBUG ] 20:04:54 DEBUG } 20:04:54 DEBUG } 20:04:54 DEBUG 20:04:54 DEBUG 20:04:54 DEBUG 20:04:54 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:54 DEBUG }, 20:04:54 DEBUG "value": [ 20:04:54 DEBUG 1713639894, 20:04:54 DEBUG "2832.8" 20:04:54 DEBUG ] 20:04:54 DEBUG } 20:04:54 DEBUG ] 20:04:54 DEBUG } 20:04:54 DEBUG } 20:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639895" 20:04:55 INFO [http_cmd]: http status code OK 20:04:55 DEBUG --- status code --- 20:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:55 DEBUG --- http response --- 20:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:55 DEBUG ----- output ----- 20:04:55 DEBUG { 20:04:55 DEBUG "status": "success", 20:04:55 DEBUG "data": { 20:04:55 DEBUG "resultType": "vector", 20:04:55 DEBUG "result": [ 20:04:55 DEBUG { 20:04:55 DEBUG "metric": { 20:04:55 DEBUG "pod": "ds-idrepo-1" 20:04:55 DEBUG }, 20:04:55 DEBUG "value": [ 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG [...] 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG "pod": "ds-cts-0" 20:04:55 DEBUG }, 20:04:55 DEBUG "value": [ 20:04:55 DEBUG 1713639895, 20:04:55 DEBUG "63438003.67635203" 20:04:55 DEBUG ] 20:04:55 DEBUG } 20:04:55 DEBUG ] 20:04:55 DEBUG } 20:04:55 DEBUG } 20:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639895" 20:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639895" 20:04:55 INFO [http_cmd]: http status code OK 20:04:55 DEBUG --- status code --- 20:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:55 DEBUG --- http response --- 20:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:55 DEBUG ----- output ----- 20:04:55 DEBUG { 20:04:55 DEBUG "status": "success", 20:04:55 DEBUG "data": { 20:04:55 DEBUG "resultType": "vector", 20:04:55 DEBUG "result": [ 20:04:55 DEBUG { 20:04:55 DEBUG "metric": { 20:04:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:55 DEBUG }, 20:04:55 DEBUG "value": [ 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG [...] 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:55 DEBUG }, 20:04:55 DEBUG "value": [ 20:04:55 DEBUG 1713639895, 20:04:55 DEBUG "2837.1" 20:04:55 DEBUG ] 20:04:55 DEBUG } 20:04:55 DEBUG ] 20:04:55 DEBUG } 20:04:55 DEBUG } 20:04:55 INFO [http_cmd]: http status code OK 20:04:55 DEBUG --- status code --- 20:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:55 DEBUG --- http response --- 20:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:55 DEBUG ----- output ----- 20:04:55 DEBUG { 20:04:55 DEBUG "status": "success", 20:04:55 DEBUG "data": { 20:04:55 DEBUG "resultType": "vector", 20:04:55 DEBUG "result": [ 20:04:55 DEBUG { 20:04:55 DEBUG "metric": { 20:04:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:04:55 DEBUG }, 20:04:55 DEBUG "value": [ 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG [...] 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:04:55 DEBUG }, 20:04:55 DEBUG "value": [ 20:04:55 DEBUG 1713639895, 20:04:55 DEBUG "9.54696654794782" 20:04:55 DEBUG ] 20:04:55 DEBUG } 20:04:55 DEBUG ] 20:04:55 DEBUG } 20:04:55 DEBUG } 20:04:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639895" 20:04:55 INFO [http_cmd]: http status code OK 20:04:55 DEBUG --- status code --- 20:04:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:55 DEBUG --- http response --- 20:04:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:55 DEBUG ----- output ----- 20:04:55 DEBUG { 20:04:55 DEBUG "status": "success", 20:04:55 DEBUG "data": { 20:04:55 DEBUG "resultType": "vector", 20:04:55 DEBUG "result": [ 20:04:55 DEBUG { 20:04:55 DEBUG "metric": { 20:04:55 DEBUG "instance": "10.142.0.107:9100" 20:04:55 DEBUG }, 20:04:55 DEBUG "value": [ 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG [...] 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG 20:04:55 DEBUG "instance": "10.142.0.43:9100" 20:04:55 DEBUG }, 20:04:55 DEBUG "value": [ 20:04:55 DEBUG 1713639895, 20:04:55 DEBUG "3661991724032" 20:04:55 DEBUG ] 20:04:55 DEBUG } 20:04:55 DEBUG ] 20:04:55 DEBUG } 20:04:55 DEBUG } 20:04:55 INFO 20:04:55 INFO [loop_until]: kubectl --namespace=xlou top pods 20:04:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:04:55 INFO [loop_until]: OK (rc = 0) 20:04:55 DEBUG --- stdout --- 20:04:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17784m 8653Mi am-767f6b67b5-qcqz2 18484m 9068Mi am-767f6b67b5-vbltz 20035m 8669Mi ds-cts-0 19920m 10844Mi ds-cts-1 18599m 10746Mi ds-cts-2 20622m 10967Mi ds-idrepo-0 7945m 3893Mi ds-idrepo-1 7758m 3882Mi ds-idrepo-2 7971m 3770Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 16m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9390m 803Mi 20:04:55 DEBUG --- stderr --- 20:04:55 DEBUG 20:04:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639896" 20:04:56 INFO [http_cmd]: http status code OK 20:04:56 DEBUG --- status code --- 20:04:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:56 DEBUG --- http response --- 20:04:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:56 DEBUG ----- output ----- 20:04:56 DEBUG { 20:04:56 DEBUG "status": "success", 20:04:56 DEBUG "data": { 20:04:56 DEBUG "resultType": "vector", 20:04:56 DEBUG "result": [ 20:04:56 DEBUG { 20:04:56 DEBUG "metric": { 20:04:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:56 DEBUG }, 20:04:56 DEBUG "value": [ 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG [...] 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:56 DEBUG }, 20:04:56 DEBUG "value": [ 20:04:56 DEBUG 1713639896, 20:04:56 DEBUG "2835.3333333333335" 20:04:56 DEBUG ] 20:04:56 DEBUG } 20:04:56 DEBUG ] 20:04:56 DEBUG } 20:04:56 DEBUG } 20:04:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639896" 20:04:56 INFO [http_cmd]: http status code OK 20:04:56 DEBUG --- status code --- 20:04:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:56 DEBUG --- http response --- 20:04:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:56 DEBUG ----- output ----- 20:04:56 DEBUG { 20:04:56 DEBUG "status": "success", 20:04:56 DEBUG "data": { 20:04:56 DEBUG "resultType": "vector", 20:04:56 DEBUG "result": [ 20:04:56 DEBUG { 20:04:56 DEBUG "metric": { 20:04:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:56 DEBUG }, 20:04:56 DEBUG "value": [ 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG [...] 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:56 DEBUG }, 20:04:56 DEBUG "value": [ 20:04:56 DEBUG 1713639896, 20:04:56 DEBUG "2835.7" 20:04:56 DEBUG ] 20:04:56 DEBUG } 20:04:56 DEBUG ] 20:04:56 DEBUG } 20:04:56 DEBUG } 20:04:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639896" 20:04:56 INFO [http_cmd]: http status code OK 20:04:56 DEBUG --- status code --- 20:04:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:56 DEBUG --- http response --- 20:04:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:04:56 DEBUG ----- output ----- 20:04:56 DEBUG { 20:04:56 DEBUG "status": "success", 20:04:56 DEBUG "data": { 20:04:56 DEBUG "resultType": "vector", 20:04:56 DEBUG "result": [ 20:04:56 DEBUG { 20:04:56 DEBUG "metric": { 20:04:56 DEBUG "pod": "ds-idrepo-1" 20:04:56 DEBUG }, 20:04:56 DEBUG "value": [ 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG [...] 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG "pod": "ds-cts-0" 20:04:56 DEBUG }, 20:04:56 DEBUG "value": [ 20:04:56 DEBUG 1713639896, 20:04:56 DEBUG "20.1964430004205" 20:04:56 DEBUG ] 20:04:56 DEBUG } 20:04:56 DEBUG ] 20:04:56 DEBUG } 20:04:56 DEBUG } 20:04:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639896" 20:04:56 INFO [http_cmd]: http status code OK 20:04:56 DEBUG --- status code --- 20:04:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:56 DEBUG --- http response --- 20:04:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:56 DEBUG ----- output ----- 20:04:56 DEBUG { 20:04:56 DEBUG "status": "success", 20:04:56 DEBUG "data": { 20:04:56 DEBUG "resultType": "vector", 20:04:56 DEBUG "result": [ 20:04:56 DEBUG { 20:04:56 DEBUG "metric": { 20:04:56 DEBUG "pod": "ds-idrepo-0" 20:04:56 DEBUG }, 20:04:56 DEBUG "value": [ 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG [...] 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG 20:04:56 DEBUG "pod": "ds-idrepo-2" 20:04:56 DEBUG }, 20:04:56 DEBUG "value": [ 20:04:56 DEBUG 1713639896, 20:04:56 DEBUG "0" 20:04:56 DEBUG ] 20:04:56 DEBUG } 20:04:56 DEBUG ] 20:04:56 DEBUG } 20:04:56 DEBUG } 20:04:57 INFO 20:04:57 INFO [loop_until]: kubectl --namespace=xlou top node 20:04:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639897" 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG { 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG "data": { 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG "result": [ 20:04:57 DEBUG { 20:04:57 DEBUG "metric": { 20:04:57 DEBUG "pod": "ds-cts-0" 20:04:57 DEBUG }, 20:04:57 DEBUG "value": [ 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG [...] 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG "pod": "ds-cts-2" 20:04:57 DEBUG }, 20:04:57 DEBUG "value": [ 20:04:57 DEBUG 1713639897, 20:04:57 DEBUG "0" 20:04:57 DEBUG ] 20:04:57 DEBUG } 20:04:57 DEBUG ] 20:04:57 DEBUG } 20:04:57 DEBUG } 20:04:57 INFO [loop_until]: OK (rc = 0) 20:04:57 DEBUG --- stdout --- 20:04:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6211Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8170m 27% 7644Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18260m 61% 11377Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 90m 0% 3345Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20102m 67% 14027Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5195Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18575m 62% 11951Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15710m 52% 2964Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20909m 70% 10891Mi 9% gke-xlou-cdm-ds-0f601751-j488 8756m 29% 7176Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19108m 64% 15794Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8633m 28% 7853Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20472m 68% 13614Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9766m 61% 6395Mi 10% 20:04:57 DEBUG --- stderr --- 20:04:57 DEBUG 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639897" 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG { 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG "data": { 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG "result": [ 20:04:57 DEBUG { 20:04:57 DEBUG "metric": { 20:04:57 DEBUG "pod": "ds-idrepo-0" 20:04:57 DEBUG }, 20:04:57 DEBUG "value": [ 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG [...] 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG "pod": "ds-cts-2" 20:04:57 DEBUG }, 20:04:57 DEBUG "value": [ 20:04:57 DEBUG 1713639897, 20:04:57 DEBUG "0" 20:04:57 DEBUG ] 20:04:57 DEBUG } 20:04:57 DEBUG ] 20:04:57 DEBUG } 20:04:57 DEBUG } 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639897" 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG { 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG "data": { 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG "result": [ 20:04:57 DEBUG { 20:04:57 DEBUG "metric": { 20:04:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:57 DEBUG }, 20:04:57 DEBUG "value": [ 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG [...] 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:57 DEBUG }, 20:04:57 DEBUG "value": [ 20:04:57 DEBUG 1713639897, 20:04:57 DEBUG "509702.5" 20:04:57 DEBUG ] 20:04:57 DEBUG } 20:04:57 DEBUG ] 20:04:57 DEBUG } 20:04:57 DEBUG } 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639897" 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG { 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG "data": { 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG "result": [ 20:04:57 DEBUG { 20:04:57 DEBUG "metric": { 20:04:57 DEBUG "instance": "10.142.0.107:9100" 20:04:57 DEBUG }, 20:04:57 DEBUG "value": [ 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG [...] 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG 20:04:57 DEBUG "instance": "10.142.0.43:9100" 20:04:57 DEBUG }, 20:04:57 DEBUG "value": [ 20:04:57 DEBUG 1713639897, 20:04:57 DEBUG "0.24586666666671325" 20:04:57 DEBUG ] 20:04:57 DEBUG } 20:04:57 DEBUG ] 20:04:57 DEBUG } 20:04:57 DEBUG } 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639897" 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639897" 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639897" 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639897" 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639897" 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639897" 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639897" 20:04:57 DEBUG ----- output ----- 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639897" 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639897" 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639897" 20:04:57 DEBUG { 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 DEBUG "data": { 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG --- http response --- 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG "result": [ 20:04:57 DEBUG { 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:57 DEBUG --- status code --- 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:04:57 DEBUG { 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG "metric": { 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG { 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG "pod": "ds-idrepo-1" 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG { 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG "status": "success", 20:04:57 INFO [http_cmd]: http status code OK 20:04:57 DEBUG { 20:04:57 DEBUG "data": { 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:04:57 DEBUG { 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG "data": { 20:04:57 DEBUG --- status code --- 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG "result": [ 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG "data": { 20:04:57 DEBUG }, 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:04:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:04:57 DEBUG "data": { 20:04:57 DEBUG { 20:04:57 DEBUG { 20:04:57 DEBUG "data": { 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG "value": [ 20:04:57 DEBUG { 20:04:57 DEBUG "result": [ 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG --- http response --- 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG "metric": { 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG "result": [ 20:04:57 DEBUG 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG { 20:04:57 DEBUG { 20:04:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:04:57 DEBUG ----- output ----- 20:04:57 DEBUG { 20:04:57 DEBUG "pod": "ds-cts-0" 20:04:57 DEBUG "result": [ 20:04:57 DEBUG { 20:04:57 DEBUG 20:04:57 DEBUG "data": { 20:04:57 DEBUG "metric": { 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG "data": { 20:04:57 DEBUG "status": "success", 20:04:57 DEBUG "result": [ 20:04:57 DEBUG }, 20:04:57 DEBUG { 20:04:57 DEBUG "metric": { 20:04:57 DEBUG 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG "instance": "10.142.0.107:9100" 20:04:57 DEBUG "data": { 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG "data": { 20:04:57 DEBUG { 20:04:57 DEBUG "value": [ 20:04:57 DEBUG "metric": { 20:04:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:04:57 DEBUG [...] 20:04:57 DEBUG "result": [ 20:04:57 DEBUG }, 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG "result": [ 20:04:57 DEBUG "resultType": "vector", 20:04:57 DEBUG "metric": { 20:04:57 DEBUG 20:04:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:57 DEBUG }, 20:04:57 DEBUG 20:04:57 DEBUG { 20:04:57 DEBUG "value": [ 20:04:57 DEBUG "result": [ 20:04:57 DEBUG { 20:04:57 DEBUG "result": [ 20:04:57 DEBUG "pod": "ds-idrepo-1" 20:04:57 DEBUG 20:04:57 DEBUG }, 20:04:57 DEBUG "value": [ 20:04:57 DEBUG 20:04:57 DEBUG "metric": { 20:04:57 DEBUG 20:04:57 DEBUG { 20:04:57 DEBUG "metric": { 20:04:57 DEBUG { 20:04:57 DEBUG }, 20:04:58 DEBUG 20:04:58 DEBUG "value": [ 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG "instance": "10.142.0.107:9100" 20:04:58 DEBUG 20:04:58 DEBUG "metric": { 20:04:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:04:58 DEBUG "metric": { 20:04:58 DEBUG "value": [ 20:04:58 DEBUG [...] 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG "pod": "ds-cts-0" 20:04:58 DEBUG }, 20:04:58 DEBUG 20:04:58 DEBUG "pod": "ds-cts-0" 20:04:58 DEBUG }, 20:04:58 DEBUG "instance": "10.142.0.107:9100" 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG }, 20:04:58 DEBUG "value": [ 20:04:58 DEBUG [...] 20:04:58 DEBUG }, 20:04:58 DEBUG "value": [ 20:04:58 DEBUG }, 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG [...] 20:04:58 DEBUG "value": [ 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG "value": [ 20:04:58 DEBUG 20:04:58 DEBUG "value": [ 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG [...] 20:04:58 DEBUG 20:04:58 DEBUG 1713639897, 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG [...] 20:04:58 DEBUG "pod": "ds-cts-2" 20:04:58 DEBUG }, 20:04:58 DEBUG 20:04:58 DEBUG "37180225.09593989" 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG "value": [ 20:04:58 DEBUG 20:04:58 DEBUG ] 20:04:58 DEBUG 20:04:58 DEBUG [...] 20:04:58 DEBUG "instance": "10.142.0.43:9100" 20:04:58 DEBUG [...] 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 1713639897, 20:04:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:04:58 DEBUG } 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG }, 20:04:58 DEBUG 20:04:58 DEBUG [...] 20:04:58 DEBUG [...] 20:04:58 DEBUG 20:04:58 DEBUG "0.0006733177777777777" 20:04:58 DEBUG }, 20:04:58 DEBUG ] 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG "value": [ 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG "pod": "ds-cts-0" 20:04:58 DEBUG ] 20:04:58 DEBUG "value": [ 20:04:58 DEBUG } 20:04:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:58 DEBUG 20:04:58 DEBUG 1713639897, 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG }, 20:04:58 DEBUG } 20:04:58 DEBUG 1713639897, 20:04:58 DEBUG } 20:04:58 DEBUG }, 20:04:58 DEBUG "instance": "10.142.0.43:9100" 20:04:58 DEBUG "86177569.3" 20:04:58 DEBUG "pod": "ds-cts-2" 20:04:58 DEBUG 20:04:58 DEBUG 20:04:58 DEBUG "value": [ 20:04:58 DEBUG ] 20:04:58 DEBUG "940191744" 20:04:58 DEBUG "value": [ 20:04:58 DEBUG }, 20:04:58 DEBUG ] 20:04:58 DEBUG }, 20:04:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:04:58 DEBUG "instance": "10.142.0.43:9100" 20:04:58 DEBUG 1713639897, 20:04:58 DEBUG } 20:04:58 DEBUG ] 20:04:58 DEBUG 1713639897, 20:04:58 DEBUG "value": [ 20:04:58 DEBUG } 20:04:58 DEBUG "value": [ 20:04:58 DEBUG }, 20:04:58 DEBUG }, 20:04:58 DEBUG "366713589.4740237" 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG "0" 20:04:58 DEBUG 1713639897, 20:04:58 DEBUG ] 20:04:58 DEBUG 1713639897, 20:04:58 DEBUG "value": [ 20:04:58 DEBUG "value": [ 20:04:58 DEBUG ] 20:04:58 DEBUG ] 20:04:58 DEBUG ] 20:04:58 DEBUG "86181446.76666667" 20:04:58 DEBUG } 20:04:58 DEBUG "31.866666666666667" 20:04:58 DEBUG 1713639897, 20:04:58 DEBUG 1713639897, 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG ] 20:04:58 DEBUG } 20:04:58 DEBUG ] 20:04:58 DEBUG "0.157" 20:04:58 DEBUG "0.08200000000000879" 20:04:58 DEBUG ] 20:04:58 DEBUG } 20:04:58 DEBUG ] 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG ] 20:04:58 DEBUG ] 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG ] 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG ] 20:04:58 DEBUG ] 20:04:58 DEBUG ] 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG } 20:04:58 DEBUG } 20:05:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639953" 20:05:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713639953" 20:05:53 ERROR [http_cmd]: ERROR 20:05:53 INFO [http_cmd]: http status code OK 20:05:53 ERROR --- status code --- 20:05:53 DEBUG --- status code --- 20:05:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:05:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:53 ERROR --- http response --- 20:05:53 DEBUG --- http response --- 20:05:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:05:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:05:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:05:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639954" 20:05:54 INFO [http_cmd]: http status code OK 20:05:54 DEBUG --- status code --- 20:05:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:54 DEBUG --- http response --- 20:05:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:54 DEBUG ----- output ----- 20:05:54 DEBUG { 20:05:54 DEBUG "status": "success", 20:05:54 DEBUG "data": { 20:05:54 DEBUG "resultType": "vector", 20:05:54 DEBUG "result": [ 20:05:54 DEBUG { 20:05:54 DEBUG "metric": { 20:05:54 DEBUG "pod": "ds-idrepo-0" 20:05:54 DEBUG }, 20:05:54 DEBUG "value": [ 20:05:54 DEBUG 20:05:54 DEBUG 20:05:54 DEBUG 20:05:54 DEBUG [...] 20:05:54 DEBUG 20:05:54 DEBUG 20:05:54 DEBUG 20:05:54 DEBUG "pod": "ds-idrepo-2" 20:05:54 DEBUG }, 20:05:54 DEBUG "value": [ 20:05:54 DEBUG 1713639954, 20:05:54 DEBUG "0" 20:05:54 DEBUG ] 20:05:54 DEBUG } 20:05:54 DEBUG ] 20:05:54 DEBUG } 20:05:54 DEBUG } 20:05:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639954" 20:05:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639954" 20:05:54 INFO [http_cmd]: http status code OK 20:05:54 DEBUG --- status code --- 20:05:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:54 DEBUG --- http response --- 20:05:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:54 DEBUG ----- output ----- 20:05:54 DEBUG { 20:05:54 DEBUG "status": "success", 20:05:54 DEBUG "data": { 20:05:54 DEBUG "resultType": "vector", 20:05:54 DEBUG "result": [ 20:05:54 DEBUG { 20:05:54 DEBUG "metric": { 20:05:54 DEBUG "pod": "ds-idrepo-0" 20:05:54 DEBUG }, 20:05:54 DEBUG "value": [ 20:05:54 DEBUG 20:05:54 DEBUG 20:05:54 DEBUG 20:05:54 DEBUG [...] 20:05:54 DEBUG 20:05:54 DEBUG 20:05:54 DEBUG 20:05:54 DEBUG "pod": "ds-idrepo-2" 20:05:54 DEBUG }, 20:05:54 DEBUG "value": [ 20:05:54 DEBUG 1713639954, 20:05:54 DEBUG "0" 20:05:54 DEBUG ] 20:05:54 DEBUG } 20:05:54 DEBUG ] 20:05:54 DEBUG } 20:05:54 DEBUG } 20:05:54 INFO [http_cmd]: http status code OK 20:05:54 DEBUG --- status code --- 20:05:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:54 DEBUG --- http response --- 20:05:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:54 DEBUG ----- output ----- 20:05:54 DEBUG { 20:05:54 DEBUG "status": "success", 20:05:54 DEBUG "data": { 20:05:54 DEBUG "resultType": "vector", 20:05:54 DEBUG "result": [ 20:05:54 DEBUG { 20:05:54 DEBUG "metric": { 20:05:54 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:54 DEBUG }, 20:05:54 DEBUG "value": [ 20:05:54 DEBUG 20:05:54 DEBUG 20:05:54 DEBUG 20:05:54 DEBUG [...] 20:05:54 DEBUG 20:05:54 DEBUG 20:05:54 DEBUG 20:05:54 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:54 DEBUG }, 20:05:54 DEBUG "value": [ 20:05:54 DEBUG 1713639954, 20:05:54 DEBUG "2823.0333333333333" 20:05:54 DEBUG ] 20:05:54 DEBUG } 20:05:54 DEBUG ] 20:05:54 DEBUG } 20:05:54 DEBUG } 20:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639955" 20:05:55 INFO [http_cmd]: http status code OK 20:05:55 DEBUG --- status code --- 20:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:55 DEBUG --- http response --- 20:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:05:55 DEBUG ----- output ----- 20:05:55 DEBUG { 20:05:55 DEBUG "status": "success", 20:05:55 DEBUG "data": { 20:05:55 DEBUG "resultType": "vector", 20:05:55 DEBUG "result": [ 20:05:55 DEBUG { 20:05:55 DEBUG "metric": { 20:05:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:05:55 DEBUG }, 20:05:55 DEBUG "value": [ 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG [...] 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:05:55 DEBUG }, 20:05:55 DEBUG "value": [ 20:05:55 DEBUG 1713639955, 20:05:55 DEBUG "0" 20:05:55 DEBUG ] 20:05:55 DEBUG } 20:05:55 DEBUG ] 20:05:55 DEBUG } 20:05:55 DEBUG } 20:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639955" 20:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713639955" 20:05:55 INFO [http_cmd]: http status code OK 20:05:55 DEBUG --- status code --- 20:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:55 DEBUG --- http response --- 20:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:55 DEBUG ----- output ----- 20:05:55 DEBUG { 20:05:55 DEBUG "status": "success", 20:05:55 DEBUG "data": { 20:05:55 DEBUG "resultType": "vector", 20:05:55 DEBUG "result": [ 20:05:55 DEBUG { 20:05:55 DEBUG "metric": { 20:05:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:55 DEBUG }, 20:05:55 DEBUG "value": [ 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG [...] 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:55 DEBUG }, 20:05:55 DEBUG "value": [ 20:05:55 DEBUG 1713639955, 20:05:55 DEBUG "2837.7666666666664" 20:05:55 DEBUG ] 20:05:55 DEBUG } 20:05:55 DEBUG ] 20:05:55 DEBUG } 20:05:55 DEBUG } 20:05:55 INFO [http_cmd]: http status code OK 20:05:55 DEBUG --- status code --- 20:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:55 DEBUG --- http response --- 20:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:55 DEBUG ----- output ----- 20:05:55 DEBUG { 20:05:55 DEBUG "status": "success", 20:05:55 DEBUG "data": { 20:05:55 DEBUG "resultType": "vector", 20:05:55 DEBUG "result": [ 20:05:55 DEBUG { 20:05:55 DEBUG "metric": { 20:05:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:05:55 DEBUG }, 20:05:55 DEBUG "value": [ 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG [...] 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:05:55 DEBUG }, 20:05:55 DEBUG "value": [ 20:05:55 DEBUG 1713639955, 20:05:55 DEBUG "9.579579095429105" 20:05:55 DEBUG ] 20:05:55 DEBUG } 20:05:55 DEBUG ] 20:05:55 DEBUG } 20:05:55 DEBUG } 20:05:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639955" 20:05:55 INFO [http_cmd]: http status code OK 20:05:55 DEBUG --- status code --- 20:05:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:55 DEBUG --- http response --- 20:05:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:55 DEBUG ----- output ----- 20:05:55 DEBUG { 20:05:55 DEBUG "status": "success", 20:05:55 DEBUG "data": { 20:05:55 DEBUG "resultType": "vector", 20:05:55 DEBUG "result": [ 20:05:55 DEBUG { 20:05:55 DEBUG "metric": { 20:05:55 DEBUG "instance": "10.142.0.107:9100" 20:05:55 DEBUG }, 20:05:55 DEBUG "value": [ 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG [...] 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG 20:05:55 DEBUG "instance": "10.142.0.43:9100" 20:05:55 DEBUG }, 20:05:55 DEBUG "value": [ 20:05:55 DEBUG 1713639955, 20:05:55 DEBUG "3666804915200" 20:05:55 DEBUG ] 20:05:55 DEBUG } 20:05:55 DEBUG ] 20:05:55 DEBUG } 20:05:55 DEBUG } 20:05:55 INFO 20:05:55 INFO [loop_until]: kubectl --namespace=xlou top pods 20:05:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:05:55 INFO [loop_until]: OK (rc = 0) 20:05:55 DEBUG --- stdout --- 20:05:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18698m 8655Mi am-767f6b67b5-qcqz2 19074m 9067Mi am-767f6b67b5-vbltz 19971m 8670Mi ds-cts-0 20002m 10847Mi ds-cts-1 18841m 10737Mi ds-cts-2 20735m 10958Mi ds-idrepo-0 7756m 3894Mi ds-idrepo-1 7678m 3882Mi ds-idrepo-2 8066m 3771Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 17m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9187m 806Mi 20:05:55 DEBUG --- stderr --- 20:05:55 DEBUG 20:05:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639956" 20:05:56 INFO [http_cmd]: http status code OK 20:05:56 DEBUG --- status code --- 20:05:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:56 DEBUG --- http response --- 20:05:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:56 DEBUG ----- output ----- 20:05:56 DEBUG { 20:05:56 DEBUG "status": "success", 20:05:56 DEBUG "data": { 20:05:56 DEBUG "resultType": "vector", 20:05:56 DEBUG "result": [ 20:05:56 DEBUG { 20:05:56 DEBUG "metric": { 20:05:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:56 DEBUG }, 20:05:56 DEBUG "value": [ 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG [...] 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:56 DEBUG }, 20:05:56 DEBUG "value": [ 20:05:56 DEBUG 1713639956, 20:05:56 DEBUG "2837.4" 20:05:56 DEBUG ] 20:05:56 DEBUG } 20:05:56 DEBUG ] 20:05:56 DEBUG } 20:05:56 DEBUG } 20:05:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639956" 20:05:56 INFO [http_cmd]: http status code OK 20:05:56 DEBUG --- status code --- 20:05:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:56 DEBUG --- http response --- 20:05:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:56 DEBUG ----- output ----- 20:05:56 DEBUG { 20:05:56 DEBUG "status": "success", 20:05:56 DEBUG "data": { 20:05:56 DEBUG "resultType": "vector", 20:05:56 DEBUG "result": [ 20:05:56 DEBUG { 20:05:56 DEBUG "metric": { 20:05:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:56 DEBUG }, 20:05:56 DEBUG "value": [ 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG [...] 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:56 DEBUG }, 20:05:56 DEBUG "value": [ 20:05:56 DEBUG 1713639956, 20:05:56 DEBUG "2837.7" 20:05:56 DEBUG ] 20:05:56 DEBUG } 20:05:56 DEBUG ] 20:05:56 DEBUG } 20:05:56 DEBUG } 20:05:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639956" 20:05:56 INFO [http_cmd]: http status code OK 20:05:56 DEBUG --- status code --- 20:05:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:56 DEBUG --- http response --- 20:05:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:05:56 DEBUG ----- output ----- 20:05:56 DEBUG { 20:05:56 DEBUG "status": "success", 20:05:56 DEBUG "data": { 20:05:56 DEBUG "resultType": "vector", 20:05:56 DEBUG "result": [ 20:05:56 DEBUG { 20:05:56 DEBUG "metric": { 20:05:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:05:56 DEBUG }, 20:05:56 DEBUG "value": [ 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG [...] 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:05:56 DEBUG }, 20:05:56 DEBUG "value": [ 20:05:56 DEBUG 1713639956, 20:05:56 DEBUG "6.786194945970162" 20:05:56 DEBUG ] 20:05:56 DEBUG } 20:05:56 DEBUG ] 20:05:56 DEBUG } 20:05:56 DEBUG } 20:05:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639956" 20:05:56 INFO [http_cmd]: http status code OK 20:05:56 DEBUG --- status code --- 20:05:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:56 DEBUG --- http response --- 20:05:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:56 DEBUG ----- output ----- 20:05:56 DEBUG { 20:05:56 DEBUG "status": "success", 20:05:56 DEBUG "data": { 20:05:56 DEBUG "resultType": "vector", 20:05:56 DEBUG "result": [ 20:05:56 DEBUG { 20:05:56 DEBUG "metric": { 20:05:56 DEBUG "pod": "ds-idrepo-0" 20:05:56 DEBUG }, 20:05:56 DEBUG "value": [ 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG [...] 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG 20:05:56 DEBUG "pod": "ds-idrepo-2" 20:05:56 DEBUG }, 20:05:56 DEBUG "value": [ 20:05:56 DEBUG 1713639956, 20:05:56 DEBUG "0" 20:05:56 DEBUG ] 20:05:56 DEBUG } 20:05:56 DEBUG ] 20:05:56 DEBUG } 20:05:56 DEBUG } 20:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639957" 20:05:57 INFO [http_cmd]: http status code OK 20:05:57 DEBUG --- status code --- 20:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:57 DEBUG --- http response --- 20:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:05:57 DEBUG ----- output ----- 20:05:57 DEBUG { 20:05:57 DEBUG "status": "success", 20:05:57 DEBUG "data": { 20:05:57 DEBUG "resultType": "vector", 20:05:57 DEBUG "result": [ 20:05:57 DEBUG { 20:05:57 DEBUG "metric": { 20:05:57 DEBUG "pod": "ds-cts-0" 20:05:57 DEBUG }, 20:05:57 DEBUG "value": [ 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG [...] 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG "pod": "ds-cts-2" 20:05:57 DEBUG }, 20:05:57 DEBUG "value": [ 20:05:57 DEBUG 1713639957, 20:05:57 DEBUG "0" 20:05:57 DEBUG ] 20:05:57 DEBUG } 20:05:57 DEBUG ] 20:05:57 DEBUG } 20:05:57 DEBUG } 20:05:57 INFO 20:05:57 INFO [loop_until]: kubectl --namespace=xlou top node 20:05:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:05:57 INFO [loop_until]: OK (rc = 0) 20:05:57 DEBUG --- stdout --- 20:05:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6207Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8221m 27% 7638Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18196m 60% 11376Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3343Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20012m 67% 14041Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 164m 0% 5197Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18507m 61% 11963Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15708m 52% 2966Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20855m 69% 10891Mi 9% gke-xlou-cdm-ds-0f601751-j488 8842m 29% 7223Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18453m 61% 15788Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8623m 28% 7860Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20256m 67% 13608Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9850m 61% 6396Mi 10% 20:05:57 DEBUG --- stderr --- 20:05:57 DEBUG 20:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713639957" 20:05:57 INFO [http_cmd]: http status code OK 20:05:57 DEBUG --- status code --- 20:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:57 DEBUG --- http response --- 20:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:05:57 DEBUG ----- output ----- 20:05:57 DEBUG { 20:05:57 DEBUG "status": "success", 20:05:57 DEBUG "data": { 20:05:57 DEBUG "resultType": "vector", 20:05:57 DEBUG "result": [ 20:05:57 DEBUG { 20:05:57 DEBUG "metric": { 20:05:57 DEBUG "pod": "ds-idrepo-0" 20:05:57 DEBUG }, 20:05:57 DEBUG "value": [ 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG [...] 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG "pod": "ds-cts-2" 20:05:57 DEBUG }, 20:05:57 DEBUG "value": [ 20:05:57 DEBUG 1713639957, 20:05:57 DEBUG "0" 20:05:57 DEBUG ] 20:05:57 DEBUG } 20:05:57 DEBUG ] 20:05:57 DEBUG } 20:05:57 DEBUG } 20:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639957" 20:05:57 INFO [http_cmd]: http status code OK 20:05:57 DEBUG --- status code --- 20:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:57 DEBUG --- http response --- 20:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:57 DEBUG ----- output ----- 20:05:57 DEBUG { 20:05:57 DEBUG "status": "success", 20:05:57 DEBUG "data": { 20:05:57 DEBUG "resultType": "vector", 20:05:57 DEBUG "result": [ 20:05:57 DEBUG { 20:05:57 DEBUG "metric": { 20:05:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:57 DEBUG }, 20:05:57 DEBUG "value": [ 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG [...] 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:57 DEBUG }, 20:05:57 DEBUG "value": [ 20:05:57 DEBUG 1713639957, 20:05:57 DEBUG "509987.5" 20:05:57 DEBUG ] 20:05:57 DEBUG } 20:05:57 DEBUG ] 20:05:57 DEBUG } 20:05:57 DEBUG } 20:05:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639957" 20:05:57 INFO [http_cmd]: http status code OK 20:05:57 DEBUG --- status code --- 20:05:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:57 DEBUG --- http response --- 20:05:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:57 DEBUG ----- output ----- 20:05:57 DEBUG { 20:05:57 DEBUG "status": "success", 20:05:57 DEBUG "data": { 20:05:57 DEBUG "resultType": "vector", 20:05:57 DEBUG "result": [ 20:05:57 DEBUG { 20:05:57 DEBUG "metric": { 20:05:57 DEBUG "instance": "10.142.0.107:9100" 20:05:57 DEBUG }, 20:05:57 DEBUG "value": [ 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG [...] 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG 20:05:57 DEBUG "instance": "10.142.0.43:9100" 20:05:57 DEBUG }, 20:05:57 DEBUG "value": [ 20:05:57 DEBUG 1713639957, 20:05:57 DEBUG "0.2522333333333364" 20:05:57 DEBUG ] 20:05:57 DEBUG } 20:05:57 DEBUG ] 20:05:57 DEBUG } 20:05:57 DEBUG } 20:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639958" 20:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639958" 20:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639958" 20:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713639958" 20:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639958" 20:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639958" 20:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639958" 20:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713639958" 20:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713639958" 20:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713639958" 20:05:58 INFO [http_cmd]: http status code OK 20:05:58 INFO [http_cmd]: http status code OK 20:05:58 INFO [http_cmd]: http status code OK 20:05:58 DEBUG --- status code --- 20:05:58 DEBUG --- status code --- 20:05:58 DEBUG --- status code --- 20:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:58 INFO [http_cmd]: http status code OK 20:05:58 INFO [http_cmd]: http status code OK 20:05:58 DEBUG --- status code --- 20:05:58 INFO [http_cmd]: http status code OK 20:05:58 INFO [http_cmd]: http status code OK 20:05:58 INFO [http_cmd]: http status code OK 20:05:58 DEBUG --- http response --- 20:05:58 DEBUG --- http response --- 20:05:58 DEBUG --- status code --- 20:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:58 DEBUG --- status code --- 20:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:58 DEBUG --- status code --- 20:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:58 DEBUG --- http response --- 20:05:58 DEBUG --- http response --- 20:05:58 DEBUG --- http response --- 20:05:58 DEBUG --- status code --- 20:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:58 INFO [http_cmd]: http status code OK 20:05:58 DEBUG --- status code --- 20:05:58 DEBUG ----- output ----- 20:05:58 DEBUG ----- output ----- 20:05:58 DEBUG { 20:05:58 DEBUG "status": "success", 20:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:58 DEBUG --- http response --- 20:05:58 DEBUG --- http response --- 20:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:58 DEBUG --- http response --- 20:05:58 DEBUG { 20:05:58 DEBUG "data": { 20:05:58 DEBUG "resultType": "vector", 20:05:58 DEBUG ----- output ----- 20:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:58 INFO [http_cmd]: http status code OK 20:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:05:58 DEBUG ----- output ----- 20:05:58 DEBUG --- http response --- 20:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:58 DEBUG "status": "success", 20:05:58 DEBUG "data": { 20:05:58 DEBUG "resultType": "vector", 20:05:58 DEBUG "result": [ 20:05:58 DEBUG ----- output ----- 20:05:58 DEBUG { 20:05:58 DEBUG { 20:05:58 DEBUG "status": "success", 20:05:58 DEBUG "data": { 20:05:58 DEBUG "resultType": "vector", 20:05:58 DEBUG "result": [ 20:05:58 DEBUG { 20:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:05:58 DEBUG { 20:05:58 DEBUG "status": "success", 20:05:58 DEBUG --- status code --- 20:05:58 DEBUG { 20:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:05:58 DEBUG "result": [ 20:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:58 DEBUG "metric": { 20:05:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:05:58 DEBUG ----- output ----- 20:05:58 DEBUG "status": "success", 20:05:58 DEBUG "data": { 20:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:05:58 DEBUG "metric": { 20:05:58 DEBUG ----- output ----- 20:05:58 DEBUG { 20:05:58 DEBUG ----- output ----- 20:05:58 DEBUG }, 20:05:58 DEBUG ----- output ----- 20:05:58 DEBUG { 20:05:58 DEBUG "resultType": "vector", 20:05:58 DEBUG --- http response --- 20:05:58 DEBUG "pod": "ds-cts-0" 20:05:58 DEBUG { 20:05:58 DEBUG "status": "success", 20:05:58 DEBUG "metric": { 20:05:58 DEBUG { 20:05:58 DEBUG "value": [ 20:05:58 DEBUG { 20:05:58 DEBUG "status": "success", 20:05:58 DEBUG "result": [ 20:05:58 DEBUG }, 20:05:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:05:58 DEBUG "data": { 20:05:58 DEBUG "data": { 20:05:58 DEBUG "instance": "10.142.0.107:9100" 20:05:58 DEBUG "status": "success", 20:05:58 DEBUG 20:05:58 DEBUG "status": "success", 20:05:58 DEBUG "data": { 20:05:58 DEBUG { 20:05:58 DEBUG "value": [ 20:05:58 DEBUG ----- output ----- 20:05:58 DEBUG "resultType": "vector", 20:05:58 DEBUG "resultType": "vector", 20:05:58 DEBUG }, 20:05:58 DEBUG "value": [ 20:05:58 DEBUG 20:05:58 DEBUG "data": { 20:05:58 DEBUG "resultType": "vector", 20:05:58 DEBUG "metric": { 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG "result": [ 20:05:58 DEBUG "result": [ 20:05:58 DEBUG { 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG "resultType": "vector", 20:05:58 DEBUG "result": [ 20:05:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:05:58 DEBUG 20:05:58 DEBUG { 20:05:58 DEBUG "data": { 20:05:58 DEBUG { 20:05:58 DEBUG "status": "success", 20:05:58 DEBUG "data": { 20:05:58 DEBUG [...] 20:05:58 DEBUG "result": [ 20:05:58 DEBUG { 20:05:58 DEBUG }, 20:05:58 DEBUG [...] 20:05:58 DEBUG "metric": { 20:05:58 DEBUG "resultType": "vector", 20:05:58 DEBUG "metric": { 20:05:58 DEBUG "resultType": "vector", 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG { 20:05:58 DEBUG "metric": { 20:05:58 DEBUG "value": [ 20:05:58 DEBUG 20:05:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:58 DEBUG "result": [ 20:05:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:05:58 DEBUG "result": [ 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG "metric": { 20:05:58 DEBUG "pod": "ds-cts-0" 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG }, 20:05:58 DEBUG { 20:05:58 DEBUG }, 20:05:58 DEBUG { 20:05:58 DEBUG [...] 20:05:58 DEBUG 20:05:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:05:58 DEBUG }, 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG "value": [ 20:05:58 DEBUG "metric": { 20:05:58 DEBUG "value": [ 20:05:58 DEBUG "metric": { 20:05:58 DEBUG 20:05:58 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:05:58 DEBUG }, 20:05:58 DEBUG "value": [ 20:05:58 DEBUG 20:05:58 DEBUG "pod": "ds-cts-2" 20:05:58 DEBUG 20:05:58 DEBUG "instance": "10.142.0.107:9100" 20:05:58 DEBUG 20:05:58 DEBUG "instance": "10.142.0.107:9100" 20:05:58 DEBUG 20:05:58 DEBUG }, 20:05:58 DEBUG "value": [ 20:05:58 DEBUG 20:05:58 DEBUG [...] 20:05:58 DEBUG }, 20:05:58 DEBUG 20:05:58 DEBUG }, 20:05:58 DEBUG 20:05:58 DEBUG }, 20:05:58 DEBUG 20:05:58 DEBUG "value": [ 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG "value": [ 20:05:58 DEBUG 20:05:58 DEBUG "value": [ 20:05:58 DEBUG 20:05:58 DEBUG "value": [ 20:05:58 DEBUG "instance": "10.142.0.43:9100" 20:05:58 DEBUG 1713639958, 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG 1713639958, 20:05:58 DEBUG [...] 20:05:58 DEBUG 20:05:58 DEBUG [...] 20:05:58 DEBUG 20:05:58 DEBUG }, 20:05:58 DEBUG "0" 20:05:58 DEBUG [...] 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG "30.9" 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG ] 20:05:58 DEBUG 20:05:58 DEBUG [...] 20:05:58 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:05:58 DEBUG }, 20:05:58 DEBUG "value": [ 20:05:58 DEBUG [...] 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG "value": [ 20:05:58 DEBUG } 20:05:58 DEBUG ] 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG [...] 20:05:58 DEBUG 1713639958, 20:05:58 DEBUG ] 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG 1713639958, 20:05:58 DEBUG 20:05:58 DEBUG 20:05:58 DEBUG "instance": "10.142.0.43:9100" 20:05:58 DEBUG 20:05:58 DEBUG "86078769.63333333" 20:05:58 DEBUG } 20:05:58 DEBUG 20:05:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:58 DEBUG "36550.234526544424" 20:05:58 DEBUG ] 20:05:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:05:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:05:58 DEBUG }, 20:05:58 DEBUG "value": [ 20:05:58 DEBUG 1713639958, 20:05:58 DEBUG "86081016.36666666" 20:05:58 DEBUG }, 20:05:58 DEBUG 20:05:58 DEBUG } 20:05:58 DEBUG } 20:05:58 DEBUG }, 20:05:58 DEBUG "value": [ 20:05:58 DEBUG 20:05:58 DEBUG ] 20:05:58 DEBUG ] 20:05:58 DEBUG ] 20:05:58 DEBUG "value": [ 20:05:58 DEBUG 1713639958, 20:05:58 DEBUG "0" 20:05:58 DEBUG ] 20:05:58 DEBUG } 20:05:58 DEBUG }, 20:05:58 DEBUG 20:05:58 DEBUG } 20:05:58 DEBUG } 20:05:58 DEBUG } 20:05:58 DEBUG "pod": "ds-cts-2" 20:05:58 DEBUG } 20:05:58 DEBUG ] 20:05:58 DEBUG 1713639958, 20:05:58 DEBUG } 20:05:58 DEBUG "value": [ 20:05:58 DEBUG "instance": "10.142.0.43:9100" 20:05:58 DEBUG ] 20:05:58 DEBUG ] 20:05:58 DEBUG } 20:05:58 DEBUG }, 20:05:58 DEBUG } 20:05:58 DEBUG "942669824" 20:05:58 DEBUG ] 20:05:58 DEBUG } 20:05:58 DEBUG }, 20:05:58 DEBUG } 20:05:58 DEBUG "value": [ 20:05:58 DEBUG ] 20:05:58 DEBUG } 20:05:58 DEBUG } 20:05:58 DEBUG ] 20:05:58 DEBUG 1713639958, 20:05:58 DEBUG } 20:05:58 DEBUG 1713639958, 20:05:58 DEBUG } 20:05:58 DEBUG "value": [ 20:05:58 DEBUG } 20:05:58 DEBUG "0.157" 20:05:58 DEBUG ] 20:05:58 DEBUG } 20:05:58 DEBUG 1713639958, 20:05:58 DEBUG } 20:05:58 DEBUG } 20:05:58 DEBUG "0.055966666666666665" 20:05:58 DEBUG "0.07866666666665005" 20:05:58 DEBUG ] 20:05:58 DEBUG ] 20:05:58 DEBUG ] 20:05:58 DEBUG } 20:05:58 DEBUG } 20:05:58 DEBUG } 20:05:58 DEBUG } 20:05:58 DEBUG ] 20:05:58 DEBUG ] 20:05:58 DEBUG } 20:05:58 DEBUG } 20:05:58 DEBUG } 20:05:58 DEBUG } 20:06:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640013" 20:06:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640013" 20:06:53 ERROR [http_cmd]: ERROR 20:06:53 INFO [http_cmd]: http status code OK 20:06:53 ERROR --- status code --- 20:06:53 DEBUG --- status code --- 20:06:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:06:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:53 ERROR --- http response --- 20:06:53 DEBUG --- http response --- 20:06:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:06:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:06:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:06:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640014" 20:06:54 INFO [http_cmd]: http status code OK 20:06:54 DEBUG --- status code --- 20:06:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:54 DEBUG --- http response --- 20:06:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:54 DEBUG ----- output ----- 20:06:54 DEBUG { 20:06:54 DEBUG "status": "success", 20:06:54 DEBUG "data": { 20:06:54 DEBUG "resultType": "vector", 20:06:54 DEBUG "result": [ 20:06:54 DEBUG { 20:06:54 DEBUG "metric": { 20:06:54 DEBUG "pod": "ds-idrepo-0" 20:06:54 DEBUG }, 20:06:54 DEBUG "value": [ 20:06:54 DEBUG 20:06:54 DEBUG 20:06:54 DEBUG 20:06:54 DEBUG [...] 20:06:54 DEBUG 20:06:54 DEBUG 20:06:54 DEBUG 20:06:54 DEBUG "pod": "ds-idrepo-2" 20:06:54 DEBUG }, 20:06:54 DEBUG "value": [ 20:06:54 DEBUG 1713640014, 20:06:54 DEBUG "0" 20:06:54 DEBUG ] 20:06:54 DEBUG } 20:06:54 DEBUG ] 20:06:54 DEBUG } 20:06:54 DEBUG } 20:06:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640014" 20:06:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640014" 20:06:54 INFO [http_cmd]: http status code OK 20:06:54 INFO [http_cmd]: http status code OK 20:06:54 DEBUG --- status code --- 20:06:54 DEBUG --- status code --- 20:06:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:54 DEBUG --- http response --- 20:06:54 DEBUG --- http response --- 20:06:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:54 DEBUG ----- output ----- 20:06:54 DEBUG { 20:06:54 DEBUG "status": "success", 20:06:54 DEBUG "data": { 20:06:54 DEBUG "resultType": "vector", 20:06:54 DEBUG "result": [ 20:06:54 DEBUG { 20:06:54 DEBUG "metric": { 20:06:54 DEBUG "pod": "ds-idrepo-0" 20:06:54 DEBUG }, 20:06:54 DEBUG "value": [ 20:06:54 DEBUG 20:06:54 DEBUG 20:06:54 DEBUG 20:06:54 DEBUG [...] 20:06:54 DEBUG 20:06:54 DEBUG 20:06:54 DEBUG 20:06:54 DEBUG "pod": "ds-idrepo-2" 20:06:54 DEBUG }, 20:06:54 DEBUG "value": [ 20:06:54 DEBUG 1713640014, 20:06:54 DEBUG "0" 20:06:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:54 DEBUG ] 20:06:54 DEBUG ----- output ----- 20:06:54 DEBUG } 20:06:54 DEBUG { 20:06:54 DEBUG ] 20:06:54 DEBUG "status": "success", 20:06:54 DEBUG } 20:06:54 DEBUG "data": { 20:06:54 DEBUG } 20:06:54 DEBUG "resultType": "vector", 20:06:54 DEBUG "result": [ 20:06:54 DEBUG { 20:06:54 DEBUG "metric": { 20:06:54 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:54 DEBUG }, 20:06:54 DEBUG "value": [ 20:06:54 DEBUG 20:06:54 DEBUG 20:06:54 DEBUG 20:06:54 DEBUG [...] 20:06:54 DEBUG 20:06:54 DEBUG 20:06:54 DEBUG 20:06:54 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:54 DEBUG }, 20:06:54 DEBUG "value": [ 20:06:54 DEBUG 1713640014, 20:06:54 DEBUG "2829.0666666666666" 20:06:54 DEBUG ] 20:06:54 DEBUG } 20:06:54 DEBUG ] 20:06:54 DEBUG } 20:06:54 DEBUG } 20:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640015" 20:06:55 INFO [http_cmd]: http status code OK 20:06:55 DEBUG --- status code --- 20:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:55 DEBUG --- http response --- 20:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:06:55 DEBUG ----- output ----- 20:06:55 DEBUG { 20:06:55 DEBUG "status": "success", 20:06:55 DEBUG "data": { 20:06:55 DEBUG "resultType": "vector", 20:06:55 DEBUG "result": [ 20:06:55 DEBUG { 20:06:55 DEBUG "metric": { 20:06:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:06:55 DEBUG }, 20:06:55 DEBUG "value": [ 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG [...] 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:06:55 DEBUG }, 20:06:55 DEBUG "value": [ 20:06:55 DEBUG 1713640015, 20:06:55 DEBUG "0" 20:06:55 DEBUG ] 20:06:55 DEBUG } 20:06:55 DEBUG ] 20:06:55 DEBUG } 20:06:55 DEBUG } 20:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640015" 20:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640015" 20:06:55 INFO [http_cmd]: http status code OK 20:06:55 DEBUG --- status code --- 20:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:55 DEBUG --- http response --- 20:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:55 DEBUG ----- output ----- 20:06:55 DEBUG { 20:06:55 DEBUG "status": "success", 20:06:55 DEBUG "data": { 20:06:55 DEBUG "resultType": "vector", 20:06:55 DEBUG "result": [ 20:06:55 DEBUG { 20:06:55 DEBUG "metric": { 20:06:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:55 DEBUG }, 20:06:55 DEBUG "value": [ 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG [...] 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:55 DEBUG }, 20:06:55 DEBUG "value": [ 20:06:55 DEBUG 1713640015, 20:06:55 DEBUG "2832.3333333333335" 20:06:55 DEBUG ] 20:06:55 DEBUG } 20:06:55 DEBUG ] 20:06:55 DEBUG } 20:06:55 DEBUG } 20:06:55 INFO [http_cmd]: http status code OK 20:06:55 DEBUG --- status code --- 20:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:55 DEBUG --- http response --- 20:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:55 DEBUG ----- output ----- 20:06:55 DEBUG { 20:06:55 DEBUG "status": "success", 20:06:55 DEBUG "data": { 20:06:55 DEBUG "resultType": "vector", 20:06:55 DEBUG "result": [ 20:06:55 DEBUG { 20:06:55 DEBUG "metric": { 20:06:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:06:55 DEBUG }, 20:06:55 DEBUG "value": [ 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG [...] 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:06:55 DEBUG }, 20:06:55 DEBUG "value": [ 20:06:55 DEBUG 1713640015, 20:06:55 DEBUG "9.581877451963376" 20:06:55 DEBUG ] 20:06:55 DEBUG } 20:06:55 DEBUG ] 20:06:55 DEBUG } 20:06:55 DEBUG } 20:06:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640015" 20:06:55 INFO [http_cmd]: http status code OK 20:06:55 DEBUG --- status code --- 20:06:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:55 DEBUG --- http response --- 20:06:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:55 DEBUG ----- output ----- 20:06:55 DEBUG { 20:06:55 DEBUG "status": "success", 20:06:55 DEBUG "data": { 20:06:55 DEBUG "resultType": "vector", 20:06:55 DEBUG "result": [ 20:06:55 DEBUG { 20:06:55 DEBUG "metric": { 20:06:55 DEBUG "instance": "10.142.0.107:9100" 20:06:55 DEBUG }, 20:06:55 DEBUG "value": [ 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG [...] 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG 20:06:55 DEBUG "instance": "10.142.0.43:9100" 20:06:55 DEBUG }, 20:06:55 DEBUG "value": [ 20:06:55 DEBUG 1713640015, 20:06:55 DEBUG "3671326241792" 20:06:55 DEBUG ] 20:06:55 DEBUG } 20:06:55 DEBUG ] 20:06:55 DEBUG } 20:06:55 DEBUG } 20:06:55 INFO 20:06:55 INFO [loop_until]: kubectl --namespace=xlou top pods 20:06:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:06:56 INFO [loop_until]: OK (rc = 0) 20:06:56 DEBUG --- stdout --- 20:06:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18563m 8656Mi am-767f6b67b5-qcqz2 18480m 9069Mi am-767f6b67b5-vbltz 19967m 8674Mi ds-cts-0 20487m 10846Mi ds-cts-1 18294m 10773Mi ds-cts-2 20900m 10956Mi ds-idrepo-0 7842m 3893Mi ds-idrepo-1 7918m 3882Mi ds-idrepo-2 8020m 3828Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 11m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9427m 807Mi 20:06:56 DEBUG --- stderr --- 20:06:56 DEBUG 20:06:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640016" 20:06:56 INFO [http_cmd]: http status code OK 20:06:56 DEBUG --- status code --- 20:06:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:56 DEBUG --- http response --- 20:06:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:56 DEBUG ----- output ----- 20:06:56 DEBUG { 20:06:56 DEBUG "status": "success", 20:06:56 DEBUG "data": { 20:06:56 DEBUG "resultType": "vector", 20:06:56 DEBUG "result": [ 20:06:56 DEBUG { 20:06:56 DEBUG "metric": { 20:06:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:56 DEBUG }, 20:06:56 DEBUG "value": [ 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG [...] 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:56 DEBUG }, 20:06:56 DEBUG "value": [ 20:06:56 DEBUG 1713640016, 20:06:56 DEBUG "2833.8333333333335" 20:06:56 DEBUG ] 20:06:56 DEBUG } 20:06:56 DEBUG ] 20:06:56 DEBUG } 20:06:56 DEBUG } 20:06:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640016" 20:06:56 INFO [http_cmd]: http status code OK 20:06:56 DEBUG --- status code --- 20:06:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:56 DEBUG --- http response --- 20:06:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:56 DEBUG ----- output ----- 20:06:56 DEBUG { 20:06:56 DEBUG "status": "success", 20:06:56 DEBUG "data": { 20:06:56 DEBUG "resultType": "vector", 20:06:56 DEBUG "result": [ 20:06:56 DEBUG { 20:06:56 DEBUG "metric": { 20:06:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:56 DEBUG }, 20:06:56 DEBUG "value": [ 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG [...] 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:56 DEBUG }, 20:06:56 DEBUG "value": [ 20:06:56 DEBUG 1713640016, 20:06:56 DEBUG "2833.9333333333334" 20:06:56 DEBUG ] 20:06:56 DEBUG } 20:06:56 DEBUG ] 20:06:56 DEBUG } 20:06:56 DEBUG } 20:06:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640016" 20:06:56 INFO [http_cmd]: http status code OK 20:06:56 DEBUG --- status code --- 20:06:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:56 DEBUG --- http response --- 20:06:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:56 DEBUG ----- output ----- 20:06:56 DEBUG { 20:06:56 DEBUG "status": "success", 20:06:56 DEBUG "data": { 20:06:56 DEBUG "resultType": "vector", 20:06:56 DEBUG "result": [ 20:06:56 DEBUG { 20:06:56 DEBUG "metric": { 20:06:56 DEBUG "pod": "ds-idrepo-1" 20:06:56 DEBUG }, 20:06:56 DEBUG "value": [ 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG [...] 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:06:56 DEBUG }, 20:06:56 DEBUG "value": [ 20:06:56 DEBUG 1713640016, 20:06:56 DEBUG "0.00003247890892427909" 20:06:56 DEBUG ] 20:06:56 DEBUG } 20:06:56 DEBUG ] 20:06:56 DEBUG } 20:06:56 DEBUG } 20:06:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640016" 20:06:56 INFO [http_cmd]: http status code OK 20:06:56 DEBUG --- status code --- 20:06:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:56 DEBUG --- http response --- 20:06:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:56 DEBUG ----- output ----- 20:06:56 DEBUG { 20:06:56 DEBUG "status": "success", 20:06:56 DEBUG "data": { 20:06:56 DEBUG "resultType": "vector", 20:06:56 DEBUG "result": [ 20:06:56 DEBUG { 20:06:56 DEBUG "metric": { 20:06:56 DEBUG "pod": "ds-idrepo-0" 20:06:56 DEBUG }, 20:06:56 DEBUG "value": [ 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG [...] 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG 20:06:56 DEBUG "pod": "ds-idrepo-2" 20:06:56 DEBUG }, 20:06:56 DEBUG "value": [ 20:06:56 DEBUG 1713640016, 20:06:56 DEBUG "0" 20:06:56 DEBUG ] 20:06:56 DEBUG } 20:06:56 DEBUG ] 20:06:56 DEBUG } 20:06:56 DEBUG } 20:06:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640017" 20:06:57 INFO [http_cmd]: http status code OK 20:06:57 DEBUG --- status code --- 20:06:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:57 DEBUG --- http response --- 20:06:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:06:57 DEBUG ----- output ----- 20:06:57 DEBUG { 20:06:57 DEBUG "status": "success", 20:06:57 DEBUG "data": { 20:06:57 DEBUG "resultType": "vector", 20:06:57 DEBUG "result": [ 20:06:57 DEBUG { 20:06:57 DEBUG "metric": { 20:06:57 DEBUG "pod": "ds-cts-0" 20:06:57 DEBUG }, 20:06:57 DEBUG "value": [ 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG [...] 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG "pod": "ds-cts-2" 20:06:57 DEBUG }, 20:06:57 DEBUG "value": [ 20:06:57 DEBUG 1713640017, 20:06:57 DEBUG "0" 20:06:57 DEBUG ] 20:06:57 DEBUG } 20:06:57 DEBUG ] 20:06:57 DEBUG } 20:06:57 DEBUG } 20:06:57 INFO 20:06:57 INFO [loop_until]: kubectl --namespace=xlou top node 20:06:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:06:57 INFO [loop_until]: OK (rc = 0) 20:06:57 DEBUG --- stdout --- 20:06:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6210Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8203m 27% 7627Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18186m 60% 11369Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3344Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20222m 67% 14030Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5215Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18515m 62% 11955Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15686m 52% 2964Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20790m 69% 10898Mi 9% gke-xlou-cdm-ds-0f601751-j488 8481m 28% 7221Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18917m 63% 15809Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8660m 29% 7852Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20615m 69% 13605Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9442m 59% 6401Mi 10% 20:06:57 DEBUG --- stderr --- 20:06:57 DEBUG 20:06:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640017" 20:06:57 INFO [http_cmd]: http status code OK 20:06:57 DEBUG --- status code --- 20:06:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:57 DEBUG --- http response --- 20:06:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:06:57 DEBUG ----- output ----- 20:06:57 DEBUG { 20:06:57 DEBUG "status": "success", 20:06:57 DEBUG "data": { 20:06:57 DEBUG "resultType": "vector", 20:06:57 DEBUG "result": [ 20:06:57 DEBUG { 20:06:57 DEBUG "metric": { 20:06:57 DEBUG "pod": "ds-idrepo-0" 20:06:57 DEBUG }, 20:06:57 DEBUG "value": [ 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG [...] 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG "pod": "ds-cts-2" 20:06:57 DEBUG }, 20:06:57 DEBUG "value": [ 20:06:57 DEBUG 1713640017, 20:06:57 DEBUG "0" 20:06:57 DEBUG ] 20:06:57 DEBUG } 20:06:57 DEBUG ] 20:06:57 DEBUG } 20:06:57 DEBUG } 20:06:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640017" 20:06:57 INFO [http_cmd]: http status code OK 20:06:57 DEBUG --- status code --- 20:06:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:57 DEBUG --- http response --- 20:06:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:57 DEBUG ----- output ----- 20:06:57 DEBUG { 20:06:57 DEBUG "status": "success", 20:06:57 DEBUG "data": { 20:06:57 DEBUG "resultType": "vector", 20:06:57 DEBUG "result": [ 20:06:57 DEBUG { 20:06:57 DEBUG "metric": { 20:06:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:57 DEBUG }, 20:06:57 DEBUG "value": [ 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG [...] 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:57 DEBUG }, 20:06:57 DEBUG "value": [ 20:06:57 DEBUG 1713640017, 20:06:57 DEBUG "510521" 20:06:57 DEBUG ] 20:06:57 DEBUG } 20:06:57 DEBUG ] 20:06:57 DEBUG } 20:06:57 DEBUG } 20:06:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640017" 20:06:57 INFO [http_cmd]: http status code OK 20:06:57 DEBUG --- status code --- 20:06:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:57 DEBUG --- http response --- 20:06:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:57 DEBUG ----- output ----- 20:06:57 DEBUG { 20:06:57 DEBUG "status": "success", 20:06:57 DEBUG "data": { 20:06:57 DEBUG "resultType": "vector", 20:06:57 DEBUG "result": [ 20:06:57 DEBUG { 20:06:57 DEBUG "metric": { 20:06:57 DEBUG "instance": "10.142.0.107:9100" 20:06:57 DEBUG }, 20:06:57 DEBUG "value": [ 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG [...] 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG 20:06:57 DEBUG "instance": "10.142.0.43:9100" 20:06:57 DEBUG }, 20:06:57 DEBUG "value": [ 20:06:57 DEBUG 1713640017, 20:06:57 DEBUG "0.29529999999990464" 20:06:57 DEBUG ] 20:06:57 DEBUG } 20:06:57 DEBUG ] 20:06:57 DEBUG } 20:06:57 DEBUG } 20:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640018" 20:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640018" 20:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640018" 20:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640018" 20:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640018" 20:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640018" 20:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640018" 20:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640018" 20:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640018" 20:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640018" 20:06:58 INFO [http_cmd]: http status code OK 20:06:58 DEBUG --- status code --- 20:06:58 INFO [http_cmd]: http status code OK 20:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:58 INFO [http_cmd]: http status code OK 20:06:58 DEBUG --- status code --- 20:06:58 DEBUG --- http response --- 20:06:58 DEBUG --- status code --- 20:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:58 INFO [http_cmd]: http status code OK 20:06:58 INFO [http_cmd]: http status code OK 20:06:58 INFO [http_cmd]: http status code OK 20:06:58 INFO [http_cmd]: http status code OK 20:06:58 INFO [http_cmd]: http status code OK 20:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:06:58 DEBUG --- http response --- 20:06:58 DEBUG --- status code --- 20:06:58 DEBUG --- status code --- 20:06:58 DEBUG --- status code --- 20:06:58 DEBUG --- status code --- 20:06:58 DEBUG --- status code --- 20:06:58 DEBUG --- http response --- 20:06:58 DEBUG ----- output ----- 20:06:58 INFO [http_cmd]: http status code OK 20:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:58 DEBUG { 20:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:06:58 INFO [http_cmd]: http status code OK 20:06:58 DEBUG --- status code --- 20:06:58 DEBUG ----- output ----- 20:06:58 DEBUG --- http response --- 20:06:58 DEBUG --- http response --- 20:06:58 DEBUG --- http response --- 20:06:58 DEBUG --- http response --- 20:06:58 DEBUG --- http response --- 20:06:58 DEBUG "status": "success", 20:06:58 DEBUG ----- output ----- 20:06:58 DEBUG --- status code --- 20:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:58 DEBUG { 20:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:58 DEBUG ----- output ----- 20:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:58 DEBUG ----- output ----- 20:06:58 DEBUG "data": { 20:06:58 DEBUG { 20:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:06:58 DEBUG --- http response --- 20:06:58 DEBUG "status": "success", 20:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:58 DEBUG { 20:06:58 DEBUG ----- output ----- 20:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:06:58 DEBUG { 20:06:58 DEBUG "resultType": "vector", 20:06:58 DEBUG "status": "success", 20:06:58 DEBUG --- http response --- 20:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:06:58 DEBUG "data": { 20:06:58 DEBUG ----- output ----- 20:06:58 DEBUG "status": "success", 20:06:58 DEBUG { 20:06:58 DEBUG ----- output ----- 20:06:58 DEBUG "status": "success", 20:06:58 DEBUG "result": [ 20:06:58 DEBUG "data": { 20:06:58 DEBUG ----- output ----- 20:06:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:06:58 DEBUG "resultType": "vector", 20:06:58 DEBUG { 20:06:58 DEBUG "data": { 20:06:58 DEBUG "status": "success", 20:06:58 DEBUG { 20:06:58 DEBUG "data": { 20:06:58 DEBUG { 20:06:58 DEBUG "resultType": "vector", 20:06:58 DEBUG { 20:06:58 DEBUG ----- output ----- 20:06:58 DEBUG "result": [ 20:06:58 DEBUG "status": "success", 20:06:58 DEBUG "resultType": "vector", 20:06:58 DEBUG "data": { 20:06:58 DEBUG "status": "success", 20:06:58 DEBUG "resultType": "vector", 20:06:58 DEBUG "metric": { 20:06:58 DEBUG "result": [ 20:06:58 DEBUG "status": "success", 20:06:58 DEBUG { 20:06:58 DEBUG { 20:06:58 DEBUG "data": { 20:06:58 DEBUG "result": [ 20:06:58 DEBUG "resultType": "vector", 20:06:58 DEBUG "data": { 20:06:58 DEBUG "result": [ 20:06:58 DEBUG "pod": "ds-idrepo-1" 20:06:58 DEBUG { 20:06:58 DEBUG "data": { 20:06:58 DEBUG "status": "success", 20:06:58 DEBUG "metric": { 20:06:58 DEBUG "resultType": "vector", 20:06:58 DEBUG { 20:06:58 DEBUG "result": [ 20:06:58 DEBUG "resultType": "vector", 20:06:58 DEBUG { 20:06:58 DEBUG }, 20:06:58 DEBUG "metric": { 20:06:58 DEBUG "resultType": "vector", 20:06:58 DEBUG "data": { 20:06:58 DEBUG "pod": "ds-cts-0" 20:06:58 DEBUG "result": [ 20:06:58 DEBUG "metric": { 20:06:58 DEBUG { 20:06:58 DEBUG "result": [ 20:06:58 DEBUG "metric": { 20:06:58 DEBUG "value": [ 20:06:58 DEBUG "pod": "ds-idrepo-1" 20:06:58 DEBUG "result": [ 20:06:58 DEBUG "resultType": "vector", 20:06:58 DEBUG }, 20:06:58 DEBUG { 20:06:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:58 DEBUG "metric": { 20:06:58 DEBUG { 20:06:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:06:58 DEBUG 20:06:58 DEBUG }, 20:06:58 DEBUG { 20:06:58 DEBUG "result": [ 20:06:58 DEBUG "value": [ 20:06:58 DEBUG "metric": { 20:06:58 DEBUG }, 20:06:58 DEBUG "instance": "10.142.0.107:9100" 20:06:58 DEBUG "metric": { 20:06:58 DEBUG }, 20:06:58 DEBUG 20:06:58 DEBUG "value": [ 20:06:58 DEBUG "metric": { 20:06:58 DEBUG { 20:06:58 DEBUG 20:06:58 DEBUG "instance": "10.142.0.107:9100" 20:06:58 DEBUG "value": [ 20:06:58 DEBUG }, 20:06:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:06:58 DEBUG "value": [ 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG "pod": "ds-cts-0" 20:06:58 DEBUG "metric": { 20:06:58 DEBUG 20:06:58 DEBUG }, 20:06:58 DEBUG 20:06:58 DEBUG "value": [ 20:06:58 DEBUG }, 20:06:58 DEBUG 20:06:58 DEBUG [...] 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG "instance": "10.142.0.107:9100" 20:06:58 DEBUG 20:06:58 DEBUG "value": [ 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG "value": [ 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG }, 20:06:58 DEBUG [...] 20:06:58 DEBUG }, 20:06:58 DEBUG [...] 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG "value": [ 20:06:58 DEBUG 20:06:58 DEBUG "value": [ 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG [...] 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG [...] 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG [...] 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG "pod": "lodemon-6f4955bbbb-clfwf" 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG [...] 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG [...] 20:06:58 DEBUG 20:06:58 DEBUG }, 20:06:58 DEBUG 20:06:58 DEBUG "pod": "lodemon-6f4955bbbb-clfwf" 20:06:58 DEBUG 20:06:58 DEBUG "pod": "ds-cts-2" 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG "value": [ 20:06:58 DEBUG [...] 20:06:58 DEBUG }, 20:06:58 DEBUG "value": [ 20:06:58 DEBUG }, 20:06:58 DEBUG 20:06:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:06:58 DEBUG 1713640018, 20:06:58 DEBUG 20:06:58 DEBUG [...] 20:06:58 DEBUG 1713640018, 20:06:58 DEBUG "value": [ 20:06:58 DEBUG 20:06:58 DEBUG }, 20:06:58 DEBUG "instance": "10.142.0.43:9100" 20:06:58 DEBUG 20:06:58 DEBUG }, 20:06:58 DEBUG "0" 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG "0" 20:06:58 DEBUG 1713640018, 20:06:58 DEBUG "instance": "10.142.0.43:9100" 20:06:58 DEBUG "value": [ 20:06:58 DEBUG }, 20:06:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:06:58 DEBUG "value": [ 20:06:58 DEBUG ] 20:06:58 DEBUG 20:06:58 DEBUG 20:06:58 DEBUG ] 20:06:58 DEBUG "28.7" 20:06:58 DEBUG }, 20:06:58 DEBUG 1713640018, 20:06:58 DEBUG "value": [ 20:06:58 DEBUG }, 20:06:58 DEBUG 1713640018, 20:06:58 DEBUG } 20:06:58 DEBUG "pod": "ds-cts-2" 20:06:58 DEBUG 20:06:58 DEBUG } 20:06:58 DEBUG ] 20:06:58 DEBUG "value": [ 20:06:58 DEBUG "0.157" 20:06:58 DEBUG 1713640018, 20:06:58 DEBUG "value": [ 20:06:58 DEBUG "943394816" 20:06:58 DEBUG ] 20:06:58 DEBUG }, 20:06:58 DEBUG "instance": "10.142.0.43:9100" 20:06:58 DEBUG ] 20:06:58 DEBUG } 20:06:58 DEBUG 1713640018, 20:06:58 DEBUG ] 20:06:58 DEBUG "85717275.2" 20:06:58 DEBUG 1713640018, 20:06:58 DEBUG ] 20:06:58 DEBUG } 20:06:58 DEBUG "value": [ 20:06:58 DEBUG }, 20:06:58 DEBUG } 20:06:58 DEBUG ] 20:06:58 DEBUG "85719005.6" 20:06:58 DEBUG } 20:06:58 DEBUG ] 20:06:58 DEBUG "0" 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG 1713640018, 20:06:58 DEBUG "value": [ 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG ] 20:06:58 DEBUG ] 20:06:58 DEBUG } 20:06:58 DEBUG ] 20:06:58 DEBUG ] 20:06:58 DEBUG "0.05403333333333333" 20:06:58 DEBUG 1713640018, 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG ] 20:06:58 DEBUG ] 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG ] 20:06:58 DEBUG "0.09600000000000554" 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG ] 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG ] 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG ] 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG ] 20:06:58 DEBUG } 20:06:58 DEBUG } 20:06:58 DEBUG } 20:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640073" 20:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640073" 20:07:53 INFO [http_cmd]: http status code OK 20:07:53 DEBUG --- status code --- 20:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:53 DEBUG --- http response --- 20:07:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:07:53 ERROR [http_cmd]: ERROR 20:07:53 ERROR --- status code --- 20:07:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:07:53 ERROR --- http response --- 20:07:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:07:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:07:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640074" 20:07:54 INFO [http_cmd]: http status code OK 20:07:54 DEBUG --- status code --- 20:07:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:54 DEBUG --- http response --- 20:07:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:54 DEBUG ----- output ----- 20:07:54 DEBUG { 20:07:54 DEBUG "status": "success", 20:07:54 DEBUG "data": { 20:07:54 DEBUG "resultType": "vector", 20:07:54 DEBUG "result": [ 20:07:54 DEBUG { 20:07:54 DEBUG "metric": { 20:07:54 DEBUG "pod": "ds-idrepo-0" 20:07:54 DEBUG }, 20:07:54 DEBUG "value": [ 20:07:54 DEBUG 20:07:54 DEBUG 20:07:54 DEBUG 20:07:54 DEBUG [...] 20:07:54 DEBUG 20:07:54 DEBUG 20:07:54 DEBUG 20:07:54 DEBUG "pod": "ds-idrepo-2" 20:07:54 DEBUG }, 20:07:54 DEBUG "value": [ 20:07:54 DEBUG 1713640074, 20:07:54 DEBUG "0" 20:07:54 DEBUG ] 20:07:54 DEBUG } 20:07:54 DEBUG ] 20:07:54 DEBUG } 20:07:54 DEBUG } 20:07:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640074" 20:07:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640074" 20:07:54 INFO [http_cmd]: http status code OK 20:07:54 DEBUG --- status code --- 20:07:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:54 DEBUG --- http response --- 20:07:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:54 DEBUG ----- output ----- 20:07:54 DEBUG { 20:07:54 DEBUG "status": "success", 20:07:54 DEBUG "data": { 20:07:54 DEBUG "resultType": "vector", 20:07:54 DEBUG "result": [ 20:07:54 DEBUG { 20:07:54 DEBUG "metric": { 20:07:54 DEBUG "pod": "am-767f6b67b5-9zw22" 20:07:54 DEBUG }, 20:07:54 DEBUG "value": [ 20:07:54 DEBUG 20:07:54 DEBUG 20:07:54 DEBUG 20:07:54 DEBUG [...] 20:07:54 DEBUG 20:07:54 DEBUG 20:07:54 DEBUG 20:07:54 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:54 DEBUG }, 20:07:54 DEBUG "value": [ 20:07:54 DEBUG 1713640074, 20:07:54 DEBUG "2841.9" 20:07:54 DEBUG ] 20:07:54 DEBUG } 20:07:54 DEBUG ] 20:07:54 DEBUG } 20:07:54 DEBUG } 20:07:54 INFO [http_cmd]: http status code OK 20:07:54 DEBUG --- status code --- 20:07:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:54 DEBUG --- http response --- 20:07:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:54 DEBUG ----- output ----- 20:07:54 DEBUG { 20:07:54 DEBUG "status": "success", 20:07:54 DEBUG "data": { 20:07:54 DEBUG "resultType": "vector", 20:07:54 DEBUG "result": [ 20:07:54 DEBUG { 20:07:54 DEBUG "metric": { 20:07:54 DEBUG "pod": "ds-idrepo-0" 20:07:54 DEBUG }, 20:07:54 DEBUG "value": [ 20:07:54 DEBUG 20:07:54 DEBUG 20:07:54 DEBUG 20:07:54 DEBUG [...] 20:07:54 DEBUG 20:07:54 DEBUG 20:07:54 DEBUG 20:07:54 DEBUG "pod": "ds-idrepo-2" 20:07:54 DEBUG }, 20:07:54 DEBUG "value": [ 20:07:54 DEBUG 1713640074, 20:07:54 DEBUG "0" 20:07:54 DEBUG ] 20:07:54 DEBUG } 20:07:54 DEBUG ] 20:07:54 DEBUG } 20:07:54 DEBUG } 20:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640075" 20:07:55 INFO [http_cmd]: http status code OK 20:07:55 DEBUG --- status code --- 20:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:55 DEBUG --- http response --- 20:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:07:55 DEBUG ----- output ----- 20:07:55 DEBUG { 20:07:55 DEBUG "status": "success", 20:07:55 DEBUG "data": { 20:07:55 DEBUG "resultType": "vector", 20:07:55 DEBUG "result": [ 20:07:55 DEBUG { 20:07:55 DEBUG "metric": { 20:07:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:07:55 DEBUG }, 20:07:55 DEBUG "value": [ 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG [...] 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:07:55 DEBUG }, 20:07:55 DEBUG "value": [ 20:07:55 DEBUG 1713640075, 20:07:55 DEBUG "0" 20:07:55 DEBUG ] 20:07:55 DEBUG } 20:07:55 DEBUG ] 20:07:55 DEBUG } 20:07:55 DEBUG } 20:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640075" 20:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640075" 20:07:55 INFO [http_cmd]: http status code OK 20:07:55 DEBUG --- status code --- 20:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:55 DEBUG --- http response --- 20:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:55 DEBUG ----- output ----- 20:07:55 DEBUG { 20:07:55 DEBUG "status": "success", 20:07:55 DEBUG "data": { 20:07:55 DEBUG "resultType": "vector", 20:07:55 DEBUG "result": [ 20:07:55 DEBUG { 20:07:55 DEBUG "metric": { 20:07:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:07:55 DEBUG }, 20:07:55 DEBUG "value": [ 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG [...] 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:55 DEBUG }, 20:07:55 DEBUG "value": [ 20:07:55 DEBUG 1713640075, 20:07:55 DEBUG "2842.6666666666665" 20:07:55 DEBUG ] 20:07:55 DEBUG } 20:07:55 DEBUG ] 20:07:55 DEBUG } 20:07:55 DEBUG } 20:07:55 INFO [http_cmd]: http status code OK 20:07:55 DEBUG --- status code --- 20:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:55 DEBUG --- http response --- 20:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:55 DEBUG ----- output ----- 20:07:55 DEBUG { 20:07:55 DEBUG "status": "success", 20:07:55 DEBUG "data": { 20:07:55 DEBUG "resultType": "vector", 20:07:55 DEBUG "result": [ 20:07:55 DEBUG { 20:07:55 DEBUG "metric": { 20:07:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:07:55 DEBUG }, 20:07:55 DEBUG "value": [ 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG [...] 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:07:55 DEBUG }, 20:07:55 DEBUG "value": [ 20:07:55 DEBUG 1713640075, 20:07:55 DEBUG "9.589105856926913" 20:07:55 DEBUG ] 20:07:55 DEBUG } 20:07:55 DEBUG ] 20:07:55 DEBUG } 20:07:55 DEBUG } 20:07:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640075" 20:07:55 INFO [http_cmd]: http status code OK 20:07:55 DEBUG --- status code --- 20:07:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:55 DEBUG --- http response --- 20:07:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:55 DEBUG ----- output ----- 20:07:55 DEBUG { 20:07:55 DEBUG "status": "success", 20:07:55 DEBUG "data": { 20:07:55 DEBUG "resultType": "vector", 20:07:55 DEBUG "result": [ 20:07:55 DEBUG { 20:07:55 DEBUG "metric": { 20:07:55 DEBUG "instance": "10.142.0.107:9100" 20:07:55 DEBUG }, 20:07:55 DEBUG "value": [ 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG [...] 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG 20:07:55 DEBUG "instance": "10.142.0.43:9100" 20:07:55 DEBUG }, 20:07:55 DEBUG "value": [ 20:07:55 DEBUG 1713640075, 20:07:55 DEBUG "3675906106368" 20:07:55 DEBUG ] 20:07:55 DEBUG } 20:07:55 DEBUG ] 20:07:55 DEBUG } 20:07:55 DEBUG } 20:07:56 INFO 20:07:56 INFO [loop_until]: kubectl --namespace=xlou top pods 20:07:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:07:56 INFO [loop_until]: OK (rc = 0) 20:07:56 DEBUG --- stdout --- 20:07:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18464m 8656Mi am-767f6b67b5-qcqz2 18449m 9069Mi am-767f6b67b5-vbltz 19720m 8671Mi ds-cts-0 20457m 10854Mi ds-cts-1 18696m 10766Mi ds-cts-2 20352m 10967Mi ds-idrepo-0 7935m 3894Mi ds-idrepo-1 7976m 3882Mi ds-idrepo-2 7876m 3828Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 12m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9651m 810Mi 20:07:56 DEBUG --- stderr --- 20:07:56 DEBUG 20:07:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640076" 20:07:56 INFO [http_cmd]: http status code OK 20:07:56 DEBUG --- status code --- 20:07:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:56 DEBUG --- http response --- 20:07:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:56 DEBUG ----- output ----- 20:07:56 DEBUG { 20:07:56 DEBUG "status": "success", 20:07:56 DEBUG "data": { 20:07:56 DEBUG "resultType": "vector", 20:07:56 DEBUG "result": [ 20:07:56 DEBUG { 20:07:56 DEBUG "metric": { 20:07:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:07:56 DEBUG }, 20:07:56 DEBUG "value": [ 20:07:56 DEBUG 20:07:56 DEBUG 20:07:56 DEBUG 20:07:56 DEBUG [...] 20:07:56 DEBUG 20:07:56 DEBUG 20:07:56 DEBUG 20:07:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:56 DEBUG }, 20:07:56 DEBUG "value": [ 20:07:56 DEBUG 1713640076, 20:07:56 DEBUG "2841.7" 20:07:56 DEBUG ] 20:07:56 DEBUG } 20:07:56 DEBUG ] 20:07:56 DEBUG } 20:07:56 DEBUG } 20:07:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640076" 20:07:56 INFO [http_cmd]: http status code OK 20:07:56 DEBUG --- status code --- 20:07:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:56 DEBUG --- http response --- 20:07:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:56 DEBUG ----- output ----- 20:07:56 DEBUG { 20:07:56 DEBUG "status": "success", 20:07:56 DEBUG "data": { 20:07:56 DEBUG "resultType": "vector", 20:07:56 DEBUG "result": [ 20:07:56 DEBUG { 20:07:56 DEBUG "metric": { 20:07:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:07:56 DEBUG }, 20:07:56 DEBUG "value": [ 20:07:56 DEBUG 20:07:56 DEBUG 20:07:56 DEBUG 20:07:56 DEBUG [...] 20:07:56 DEBUG 20:07:56 DEBUG 20:07:56 DEBUG 20:07:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:56 DEBUG }, 20:07:56 DEBUG "value": [ 20:07:56 DEBUG 1713640076, 20:07:56 DEBUG "2840.4333333333334" 20:07:56 DEBUG ] 20:07:56 DEBUG } 20:07:56 DEBUG ] 20:07:56 DEBUG } 20:07:56 DEBUG } 20:07:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640076" 20:07:56 INFO [http_cmd]: http status code OK 20:07:56 DEBUG --- status code --- 20:07:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:56 DEBUG --- http response --- 20:07:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:56 DEBUG ----- output ----- 20:07:56 DEBUG { 20:07:56 DEBUG "status": "success", 20:07:56 DEBUG "data": { 20:07:56 DEBUG "resultType": "vector", 20:07:56 DEBUG "result": [ 20:07:56 DEBUG { 20:07:56 DEBUG "metric": { 20:07:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:07:56 DEBUG }, 20:07:56 DEBUG "value": [ 20:07:56 DEBUG 20:07:56 DEBUG 20:07:56 DEBUG 20:07:56 DEBUG [...] 20:07:56 DEBUG 20:07:56 DEBUG 20:07:56 DEBUG 20:07:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:07:56 DEBUG }, 20:07:56 DEBUG "value": [ 20:07:56 DEBUG 1713640076, 20:07:56 DEBUG "9.552225179910515" 20:07:56 DEBUG ] 20:07:56 DEBUG } 20:07:56 DEBUG ] 20:07:56 DEBUG } 20:07:56 DEBUG } 20:07:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640077" 20:07:57 INFO [http_cmd]: http status code OK 20:07:57 DEBUG --- status code --- 20:07:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:57 DEBUG --- http response --- 20:07:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:57 DEBUG ----- output ----- 20:07:57 DEBUG { 20:07:57 DEBUG "status": "success", 20:07:57 DEBUG "data": { 20:07:57 DEBUG "resultType": "vector", 20:07:57 DEBUG "result": [ 20:07:57 DEBUG { 20:07:57 DEBUG "metric": { 20:07:57 DEBUG "pod": "ds-idrepo-0" 20:07:57 DEBUG }, 20:07:57 DEBUG "value": [ 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG [...] 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG "pod": "ds-idrepo-2" 20:07:57 DEBUG }, 20:07:57 DEBUG "value": [ 20:07:57 DEBUG 1713640077, 20:07:57 DEBUG "0" 20:07:57 DEBUG ] 20:07:57 DEBUG } 20:07:57 DEBUG ] 20:07:57 DEBUG } 20:07:57 DEBUG } 20:07:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640077" 20:07:57 INFO [http_cmd]: http status code OK 20:07:57 DEBUG --- status code --- 20:07:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:57 DEBUG --- http response --- 20:07:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:07:57 DEBUG ----- output ----- 20:07:57 DEBUG { 20:07:57 DEBUG "status": "success", 20:07:57 DEBUG "data": { 20:07:57 DEBUG "resultType": "vector", 20:07:57 DEBUG "result": [ 20:07:57 DEBUG { 20:07:57 DEBUG "metric": { 20:07:57 DEBUG "pod": "ds-cts-0" 20:07:57 DEBUG }, 20:07:57 DEBUG "value": [ 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG [...] 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG "pod": "ds-cts-2" 20:07:57 DEBUG }, 20:07:57 DEBUG "value": [ 20:07:57 DEBUG 1713640077, 20:07:57 DEBUG "0" 20:07:57 DEBUG ] 20:07:57 DEBUG } 20:07:57 DEBUG ] 20:07:57 DEBUG } 20:07:57 DEBUG } 20:07:57 INFO 20:07:57 INFO [loop_until]: kubectl --namespace=xlou top node 20:07:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:07:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640077" 20:07:57 INFO [http_cmd]: http status code OK 20:07:57 DEBUG --- status code --- 20:07:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:57 DEBUG --- http response --- 20:07:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:07:57 DEBUG ----- output ----- 20:07:57 DEBUG { 20:07:57 DEBUG "status": "success", 20:07:57 DEBUG "data": { 20:07:57 DEBUG "resultType": "vector", 20:07:57 DEBUG "result": [ 20:07:57 DEBUG { 20:07:57 DEBUG "metric": { 20:07:57 DEBUG "pod": "ds-idrepo-0" 20:07:57 DEBUG }, 20:07:57 DEBUG "value": [ 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG [...] 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG "pod": "ds-cts-2" 20:07:57 DEBUG }, 20:07:57 DEBUG "value": [ 20:07:57 DEBUG 1713640077, 20:07:57 DEBUG "0" 20:07:57 DEBUG ] 20:07:57 DEBUG } 20:07:57 DEBUG ] 20:07:57 DEBUG } 20:07:57 DEBUG } 20:07:57 INFO [loop_until]: OK (rc = 0) 20:07:57 DEBUG --- stdout --- 20:07:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6212Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8163m 27% 7634Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17714m 59% 11378Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3345Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19562m 65% 14074Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 138m 0% 5193Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18484m 61% 11949Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15729m 52% 2964Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20937m 70% 10886Mi 9% gke-xlou-cdm-ds-0f601751-j488 8792m 29% 7223Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18929m 63% 15810Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8657m 28% 7842Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20615m 69% 13625Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9813m 61% 6396Mi 10% 20:07:57 DEBUG --- stderr --- 20:07:57 DEBUG 20:07:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640077" 20:07:57 INFO [http_cmd]: http status code OK 20:07:57 DEBUG --- status code --- 20:07:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:57 DEBUG --- http response --- 20:07:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:57 DEBUG ----- output ----- 20:07:57 DEBUG { 20:07:57 DEBUG "status": "success", 20:07:57 DEBUG "data": { 20:07:57 DEBUG "resultType": "vector", 20:07:57 DEBUG "result": [ 20:07:57 DEBUG { 20:07:57 DEBUG "metric": { 20:07:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:07:57 DEBUG }, 20:07:57 DEBUG "value": [ 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG [...] 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG 20:07:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:57 DEBUG }, 20:07:57 DEBUG "value": [ 20:07:57 DEBUG 1713640077, 20:07:57 DEBUG "511091.5" 20:07:57 DEBUG ] 20:07:57 DEBUG } 20:07:57 DEBUG ] 20:07:57 DEBUG } 20:07:57 DEBUG } 20:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640078" 20:07:58 INFO [http_cmd]: http status code OK 20:07:58 DEBUG --- status code --- 20:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:58 DEBUG --- http response --- 20:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:58 DEBUG ----- output ----- 20:07:58 DEBUG { 20:07:58 DEBUG "status": "success", 20:07:58 DEBUG "data": { 20:07:58 DEBUG "resultType": "vector", 20:07:58 DEBUG "result": [ 20:07:58 DEBUG { 20:07:58 DEBUG "metric": { 20:07:58 DEBUG "instance": "10.142.0.107:9100" 20:07:58 DEBUG }, 20:07:58 DEBUG "value": [ 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG [...] 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG "instance": "10.142.0.43:9100" 20:07:58 DEBUG }, 20:07:58 DEBUG "value": [ 20:07:58 DEBUG 1713640078, 20:07:58 DEBUG "0.21773333333323325" 20:07:58 DEBUG ] 20:07:58 DEBUG } 20:07:58 DEBUG ] 20:07:58 DEBUG } 20:07:58 DEBUG } 20:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640078" 20:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640078" 20:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640078" 20:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640078" 20:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640078" 20:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640078" 20:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640078" 20:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640078" 20:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640078" 20:07:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640078" 20:07:58 INFO [http_cmd]: http status code OK 20:07:58 INFO [http_cmd]: http status code OK 20:07:58 INFO [http_cmd]: http status code OK 20:07:58 DEBUG --- status code --- 20:07:58 INFO [http_cmd]: http status code OK 20:07:58 INFO [http_cmd]: http status code OK 20:07:58 DEBUG --- status code --- 20:07:58 DEBUG --- status code --- 20:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:58 DEBUG --- status code --- 20:07:58 INFO [http_cmd]: http status code OK 20:07:58 INFO [http_cmd]: http status code OK 20:07:58 DEBUG --- status code --- 20:07:58 INFO [http_cmd]: http status code OK 20:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:58 DEBUG --- http response --- 20:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:58 DEBUG --- status code --- 20:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:58 DEBUG --- status code --- 20:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:58 DEBUG --- status code --- 20:07:58 DEBUG --- http response --- 20:07:58 DEBUG --- http response --- 20:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:07:58 DEBUG --- http response --- 20:07:58 DEBUG --- http response --- 20:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:58 DEBUG --- http response --- 20:07:58 INFO [http_cmd]: http status code OK 20:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:07:58 INFO [http_cmd]: http status code OK 20:07:58 DEBUG ----- output ----- 20:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:07:58 DEBUG --- http response --- 20:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:58 DEBUG --- status code --- 20:07:58 DEBUG --- http response --- 20:07:58 DEBUG ----- output ----- 20:07:58 DEBUG ----- output ----- 20:07:58 DEBUG --- status code --- 20:07:58 DEBUG { 20:07:58 DEBUG ----- output ----- 20:07:58 DEBUG ----- output ----- 20:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:58 DEBUG ----- output ----- 20:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:58 DEBUG { 20:07:58 DEBUG "status": "success", 20:07:58 DEBUG { 20:07:58 DEBUG "status": "success", 20:07:58 DEBUG { 20:07:58 DEBUG { 20:07:58 DEBUG ----- output ----- 20:07:58 DEBUG { 20:07:58 DEBUG --- http response --- 20:07:58 DEBUG ----- output ----- 20:07:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:07:58 DEBUG "data": { 20:07:58 DEBUG "status": "success", 20:07:58 DEBUG "data": { 20:07:58 DEBUG "resultType": "vector", 20:07:58 DEBUG "status": "success", 20:07:58 DEBUG { 20:07:58 DEBUG "status": "success", 20:07:58 DEBUG "data": { 20:07:58 DEBUG { 20:07:58 DEBUG --- http response --- 20:07:58 DEBUG "resultType": "vector", 20:07:58 DEBUG "data": { 20:07:58 DEBUG "result": [ 20:07:58 DEBUG "status": "success", 20:07:58 DEBUG "data": { 20:07:58 DEBUG "status": "success", 20:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:07:58 DEBUG "resultType": "vector", 20:07:58 DEBUG "status": "success", 20:07:58 DEBUG "result": [ 20:07:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:07:58 DEBUG "resultType": "vector", 20:07:58 DEBUG { 20:07:58 DEBUG "data": { 20:07:58 DEBUG "resultType": "vector", 20:07:58 DEBUG "data": { 20:07:58 DEBUG ----- output ----- 20:07:58 DEBUG "result": [ 20:07:58 DEBUG "data": { 20:07:58 DEBUG { 20:07:58 DEBUG ----- output ----- 20:07:58 DEBUG "result": [ 20:07:58 DEBUG "metric": { 20:07:58 DEBUG "resultType": "vector", 20:07:58 DEBUG "result": [ 20:07:58 DEBUG "resultType": "vector", 20:07:58 DEBUG { 20:07:58 DEBUG { 20:07:58 DEBUG "resultType": "vector", 20:07:58 DEBUG "metric": { 20:07:58 DEBUG { 20:07:58 DEBUG { 20:07:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:07:58 DEBUG "result": [ 20:07:58 DEBUG { 20:07:58 DEBUG "result": [ 20:07:58 DEBUG "status": "success", 20:07:58 DEBUG "metric": { 20:07:58 DEBUG "result": [ 20:07:58 DEBUG "instance": "10.142.0.107:9100" 20:07:58 DEBUG "status": "success", 20:07:58 DEBUG "data": { 20:07:58 DEBUG }, 20:07:58 DEBUG { 20:07:58 DEBUG "metric": { 20:07:58 DEBUG { 20:07:58 DEBUG "data": { 20:07:58 DEBUG "instance": "10.142.0.107:9100" 20:07:58 DEBUG { 20:07:58 DEBUG }, 20:07:58 DEBUG "value": [ 20:07:58 DEBUG "metric": { 20:07:58 DEBUG "value": [ 20:07:58 DEBUG "metric": { 20:07:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:07:58 DEBUG "metric": { 20:07:58 DEBUG "resultType": "vector", 20:07:58 DEBUG }, 20:07:58 DEBUG "metric": { 20:07:58 DEBUG 20:07:58 DEBUG "resultType": "vector", 20:07:58 DEBUG "pod": "ds-cts-0" 20:07:58 DEBUG 20:07:58 DEBUG "pod": "ds-cts-0" 20:07:58 DEBUG }, 20:07:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:07:58 DEBUG "result": [ 20:07:58 DEBUG "value": [ 20:07:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:07:58 DEBUG 20:07:58 DEBUG "result": [ 20:07:58 DEBUG }, 20:07:58 DEBUG 20:07:58 DEBUG }, 20:07:58 DEBUG "value": [ 20:07:58 DEBUG }, 20:07:58 DEBUG { 20:07:58 DEBUG 20:07:58 DEBUG }, 20:07:58 DEBUG 20:07:58 DEBUG { 20:07:58 DEBUG "value": [ 20:07:58 DEBUG 20:07:58 DEBUG [...] 20:07:58 DEBUG 20:07:58 DEBUG "value": [ 20:07:58 DEBUG "metric": { 20:07:58 DEBUG 20:07:58 DEBUG "value": [ 20:07:58 DEBUG [...] 20:07:58 DEBUG "metric": { 20:07:58 DEBUG 20:07:58 DEBUG "value": [ 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG "instance": "10.142.0.107:9100" 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG }, 20:07:58 DEBUG [...] 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG }, 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG [...] 20:07:58 DEBUG 20:07:58 DEBUG "value": [ 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG "value": [ 20:07:58 DEBUG [...] 20:07:58 DEBUG 20:07:58 DEBUG "pod": "idm-b85f9d978-txpwv" 20:07:58 DEBUG 20:07:58 DEBUG [...] 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG "instance": "10.142.0.43:9100" 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG [...] 20:07:58 DEBUG }, 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG [...] 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG }, 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG "value": [ 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG [...] 20:07:58 DEBUG "instance": "10.142.0.43:9100" 20:07:58 DEBUG "value": [ 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 1713640078, 20:07:58 DEBUG "pod": "ds-cts-2" 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG }, 20:07:58 DEBUG 1713640078, 20:07:58 DEBUG [...] 20:07:58 DEBUG "pod": "ds-cts-2" 20:07:58 DEBUG 20:07:58 DEBUG "0" 20:07:58 DEBUG }, 20:07:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:58 DEBUG 20:07:58 DEBUG 20:07:58 DEBUG "value": [ 20:07:58 DEBUG "85994389.63333333" 20:07:58 DEBUG 20:07:58 DEBUG }, 20:07:58 DEBUG "pod": "idm-b85f9d978-txpwv" 20:07:58 DEBUG ] 20:07:58 DEBUG "value": [ 20:07:58 DEBUG }, 20:07:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:07:58 DEBUG 20:07:58 DEBUG 1713640078, 20:07:58 DEBUG ] 20:07:58 DEBUG 20:07:58 DEBUG "value": [ 20:07:58 DEBUG }, 20:07:58 DEBUG } 20:07:58 DEBUG 1713640078, 20:07:58 DEBUG "value": [ 20:07:58 DEBUG }, 20:07:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:07:58 DEBUG "85989001.1" 20:07:58 DEBUG } 20:07:58 DEBUG 20:07:58 DEBUG 1713640078, 20:07:58 DEBUG "value": [ 20:07:58 DEBUG ] 20:07:58 DEBUG "0.0006000000000000001" 20:07:58 DEBUG 1713640078, 20:07:58 DEBUG "value": [ 20:07:58 DEBUG }, 20:07:58 DEBUG ] 20:07:58 DEBUG ] 20:07:58 DEBUG "instance": "10.142.0.43:9100" 20:07:58 DEBUG "25.2" 20:07:58 DEBUG 1713640078, 20:07:58 DEBUG } 20:07:58 DEBUG ] 20:07:58 DEBUG "0.157" 20:07:58 DEBUG 1713640078, 20:07:58 DEBUG "value": [ 20:07:58 DEBUG } 20:07:58 DEBUG } 20:07:58 DEBUG }, 20:07:58 DEBUG ] 20:07:58 DEBUG "40699422.34780705" 20:07:58 DEBUG } 20:07:58 DEBUG } 20:07:58 DEBUG ] 20:07:58 DEBUG "947933184" 20:07:58 DEBUG 1713640078, 20:07:58 DEBUG ] 20:07:58 DEBUG } 20:07:58 DEBUG "value": [ 20:07:58 DEBUG } 20:07:58 DEBUG ] 20:07:58 DEBUG ] 20:07:58 DEBUG } 20:07:58 DEBUG ] 20:07:58 DEBUG "0" 20:07:58 DEBUG } 20:07:58 DEBUG 1713640078, 20:07:58 DEBUG ] 20:07:58 DEBUG } 20:07:58 DEBUG } 20:07:58 DEBUG ] 20:07:58 DEBUG } 20:07:58 DEBUG ] 20:07:58 DEBUG } 20:07:58 DEBUG "0.07266666666667068" 20:07:58 DEBUG } 20:07:58 DEBUG ] 20:07:58 DEBUG } 20:07:58 DEBUG } 20:07:58 DEBUG ] 20:07:58 DEBUG } 20:07:58 DEBUG ] 20:07:58 DEBUG } 20:07:58 DEBUG } 20:07:58 DEBUG } 20:07:58 DEBUG } 20:07:58 DEBUG ] 20:07:58 DEBUG } 20:07:58 DEBUG } 20:07:58 DEBUG } 20:07:58 DEBUG } 20:07:58 DEBUG ] 20:07:58 DEBUG } 20:07:58 DEBUG } 20:07:58 DEBUG } 20:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640133" 20:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640133" 20:08:53 INFO [http_cmd]: http status code OK 20:08:53 DEBUG --- status code --- 20:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:53 DEBUG --- http response --- 20:08:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:08:53 ERROR [http_cmd]: ERROR 20:08:53 ERROR --- status code --- 20:08:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:08:53 ERROR --- http response --- 20:08:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:08:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:08:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640134" 20:08:54 INFO [http_cmd]: http status code OK 20:08:54 DEBUG --- status code --- 20:08:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:54 DEBUG --- http response --- 20:08:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:54 DEBUG ----- output ----- 20:08:54 DEBUG { 20:08:54 DEBUG "status": "success", 20:08:54 DEBUG "data": { 20:08:54 DEBUG "resultType": "vector", 20:08:54 DEBUG "result": [ 20:08:54 DEBUG { 20:08:54 DEBUG "metric": { 20:08:54 DEBUG "pod": "ds-idrepo-0" 20:08:54 DEBUG }, 20:08:54 DEBUG "value": [ 20:08:54 DEBUG 20:08:54 DEBUG 20:08:54 DEBUG 20:08:54 DEBUG [...] 20:08:54 DEBUG 20:08:54 DEBUG 20:08:54 DEBUG 20:08:54 DEBUG "pod": "ds-idrepo-2" 20:08:54 DEBUG }, 20:08:54 DEBUG "value": [ 20:08:54 DEBUG 1713640134, 20:08:54 DEBUG "0" 20:08:54 DEBUG ] 20:08:54 DEBUG } 20:08:54 DEBUG ] 20:08:54 DEBUG } 20:08:54 DEBUG } 20:08:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640134" 20:08:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640134" 20:08:54 INFO [http_cmd]: http status code OK 20:08:54 DEBUG --- status code --- 20:08:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:54 DEBUG --- http response --- 20:08:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:54 DEBUG ----- output ----- 20:08:54 DEBUG { 20:08:54 DEBUG "status": "success", 20:08:54 DEBUG "data": { 20:08:54 DEBUG "resultType": "vector", 20:08:54 DEBUG "result": [ 20:08:54 DEBUG { 20:08:54 DEBUG "metric": { 20:08:54 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:54 DEBUG }, 20:08:54 DEBUG "value": [ 20:08:54 DEBUG 20:08:54 DEBUG 20:08:54 DEBUG 20:08:54 DEBUG [...] 20:08:54 DEBUG 20:08:54 DEBUG 20:08:54 DEBUG 20:08:54 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:54 DEBUG }, 20:08:54 DEBUG "value": [ 20:08:54 DEBUG 1713640134, 20:08:54 DEBUG "2834.366666666667" 20:08:54 DEBUG ] 20:08:54 DEBUG } 20:08:54 DEBUG ] 20:08:54 DEBUG } 20:08:54 DEBUG } 20:08:54 INFO [http_cmd]: http status code OK 20:08:54 DEBUG --- status code --- 20:08:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:54 DEBUG --- http response --- 20:08:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:54 DEBUG ----- output ----- 20:08:54 DEBUG { 20:08:54 DEBUG "status": "success", 20:08:54 DEBUG "data": { 20:08:54 DEBUG "resultType": "vector", 20:08:54 DEBUG "result": [ 20:08:54 DEBUG { 20:08:54 DEBUG "metric": { 20:08:54 DEBUG "pod": "ds-idrepo-0" 20:08:54 DEBUG }, 20:08:54 DEBUG "value": [ 20:08:54 DEBUG 20:08:54 DEBUG 20:08:54 DEBUG 20:08:54 DEBUG [...] 20:08:54 DEBUG 20:08:54 DEBUG 20:08:54 DEBUG 20:08:54 DEBUG "pod": "ds-idrepo-2" 20:08:54 DEBUG }, 20:08:54 DEBUG "value": [ 20:08:54 DEBUG 1713640134, 20:08:54 DEBUG "0" 20:08:54 DEBUG ] 20:08:54 DEBUG } 20:08:54 DEBUG ] 20:08:54 DEBUG } 20:08:54 DEBUG } 20:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640135" 20:08:55 INFO [http_cmd]: http status code OK 20:08:55 DEBUG --- status code --- 20:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:55 DEBUG --- http response --- 20:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:08:55 DEBUG ----- output ----- 20:08:55 DEBUG { 20:08:55 DEBUG "status": "success", 20:08:55 DEBUG "data": { 20:08:55 DEBUG "resultType": "vector", 20:08:55 DEBUG "result": [ 20:08:55 DEBUG { 20:08:55 DEBUG "metric": { 20:08:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:08:55 DEBUG }, 20:08:55 DEBUG "value": [ 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG [...] 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:08:55 DEBUG }, 20:08:55 DEBUG "value": [ 20:08:55 DEBUG 1713640135, 20:08:55 DEBUG "0" 20:08:55 DEBUG ] 20:08:55 DEBUG } 20:08:55 DEBUG ] 20:08:55 DEBUG } 20:08:55 DEBUG } 20:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640135" 20:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640135" 20:08:55 INFO [http_cmd]: http status code OK 20:08:55 DEBUG --- status code --- 20:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:55 DEBUG --- http response --- 20:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:55 DEBUG ----- output ----- 20:08:55 DEBUG { 20:08:55 DEBUG "status": "success", 20:08:55 DEBUG "data": { 20:08:55 DEBUG "resultType": "vector", 20:08:55 DEBUG "result": [ 20:08:55 DEBUG { 20:08:55 DEBUG "metric": { 20:08:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:55 DEBUG }, 20:08:55 DEBUG "value": [ 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG [...] 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:55 DEBUG }, 20:08:55 DEBUG "value": [ 20:08:55 DEBUG 1713640135, 20:08:55 DEBUG "2831.2666666666664" 20:08:55 DEBUG ] 20:08:55 DEBUG } 20:08:55 DEBUG ] 20:08:55 DEBUG } 20:08:55 DEBUG } 20:08:55 INFO [http_cmd]: http status code OK 20:08:55 DEBUG --- status code --- 20:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:55 DEBUG --- http response --- 20:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:55 DEBUG ----- output ----- 20:08:55 DEBUG { 20:08:55 DEBUG "status": "success", 20:08:55 DEBUG "data": { 20:08:55 DEBUG "resultType": "vector", 20:08:55 DEBUG "result": [ 20:08:55 DEBUG { 20:08:55 DEBUG "metric": { 20:08:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:08:55 DEBUG }, 20:08:55 DEBUG "value": [ 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG [...] 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:08:55 DEBUG }, 20:08:55 DEBUG "value": [ 20:08:55 DEBUG 1713640135, 20:08:55 DEBUG "9.523535220643723" 20:08:55 DEBUG ] 20:08:55 DEBUG } 20:08:55 DEBUG ] 20:08:55 DEBUG } 20:08:55 DEBUG } 20:08:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640135" 20:08:55 INFO [http_cmd]: http status code OK 20:08:55 DEBUG --- status code --- 20:08:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:55 DEBUG --- http response --- 20:08:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:55 DEBUG ----- output ----- 20:08:55 DEBUG { 20:08:55 DEBUG "status": "success", 20:08:55 DEBUG "data": { 20:08:55 DEBUG "resultType": "vector", 20:08:55 DEBUG "result": [ 20:08:55 DEBUG { 20:08:55 DEBUG "metric": { 20:08:55 DEBUG "instance": "10.142.0.107:9100" 20:08:55 DEBUG }, 20:08:55 DEBUG "value": [ 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG [...] 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG 20:08:55 DEBUG "instance": "10.142.0.43:9100" 20:08:55 DEBUG }, 20:08:55 DEBUG "value": [ 20:08:55 DEBUG 1713640135, 20:08:55 DEBUG "3680796664832" 20:08:55 DEBUG ] 20:08:55 DEBUG } 20:08:55 DEBUG ] 20:08:55 DEBUG } 20:08:55 DEBUG } 20:08:56 INFO 20:08:56 INFO [loop_until]: kubectl --namespace=xlou top pods 20:08:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:08:56 INFO [loop_until]: OK (rc = 0) 20:08:56 DEBUG --- stdout --- 20:08:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17960m 8659Mi am-767f6b67b5-qcqz2 18105m 9067Mi am-767f6b67b5-vbltz 20099m 8675Mi ds-cts-0 19524m 10970Mi ds-cts-1 19024m 10775Mi ds-cts-2 20856m 10964Mi ds-idrepo-0 7841m 3893Mi ds-idrepo-1 8074m 3882Mi ds-idrepo-2 7984m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 12m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9494m 812Mi 20:08:56 DEBUG --- stderr --- 20:08:56 DEBUG 20:08:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640136" 20:08:56 INFO [http_cmd]: http status code OK 20:08:56 DEBUG --- status code --- 20:08:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:56 DEBUG --- http response --- 20:08:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:56 DEBUG ----- output ----- 20:08:56 DEBUG { 20:08:56 DEBUG "status": "success", 20:08:56 DEBUG "data": { 20:08:56 DEBUG "resultType": "vector", 20:08:56 DEBUG "result": [ 20:08:56 DEBUG { 20:08:56 DEBUG "metric": { 20:08:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:56 DEBUG }, 20:08:56 DEBUG "value": [ 20:08:56 DEBUG 20:08:56 DEBUG 20:08:56 DEBUG 20:08:56 DEBUG [...] 20:08:56 DEBUG 20:08:56 DEBUG 20:08:56 DEBUG 20:08:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:56 DEBUG }, 20:08:56 DEBUG "value": [ 20:08:56 DEBUG 1713640136, 20:08:56 DEBUG "2832.7" 20:08:56 DEBUG ] 20:08:56 DEBUG } 20:08:56 DEBUG ] 20:08:56 DEBUG } 20:08:56 DEBUG } 20:08:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640136" 20:08:56 INFO [http_cmd]: http status code OK 20:08:56 DEBUG --- status code --- 20:08:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:56 DEBUG --- http response --- 20:08:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:56 DEBUG ----- output ----- 20:08:56 DEBUG { 20:08:56 DEBUG "status": "success", 20:08:56 DEBUG "data": { 20:08:56 DEBUG "resultType": "vector", 20:08:56 DEBUG "result": [ 20:08:56 DEBUG { 20:08:56 DEBUG "metric": { 20:08:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:56 DEBUG }, 20:08:56 DEBUG "value": [ 20:08:56 DEBUG 20:08:56 DEBUG 20:08:56 DEBUG 20:08:56 DEBUG [...] 20:08:56 DEBUG 20:08:56 DEBUG 20:08:56 DEBUG 20:08:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:56 DEBUG }, 20:08:56 DEBUG "value": [ 20:08:56 DEBUG 1713640136, 20:08:56 DEBUG "2829.5" 20:08:56 DEBUG ] 20:08:56 DEBUG } 20:08:56 DEBUG ] 20:08:56 DEBUG } 20:08:56 DEBUG } 20:08:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640136" 20:08:56 INFO [http_cmd]: http status code OK 20:08:56 DEBUG --- status code --- 20:08:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:56 DEBUG --- http response --- 20:08:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:56 DEBUG ----- output ----- 20:08:56 DEBUG { 20:08:56 DEBUG "status": "success", 20:08:56 DEBUG "data": { 20:08:56 DEBUG "resultType": "vector", 20:08:56 DEBUG "result": [ 20:08:56 DEBUG { 20:08:56 DEBUG "metric": { 20:08:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:08:56 DEBUG }, 20:08:56 DEBUG "value": [ 20:08:56 DEBUG 20:08:56 DEBUG 20:08:56 DEBUG 20:08:56 DEBUG [...] 20:08:56 DEBUG 20:08:56 DEBUG 20:08:56 DEBUG 20:08:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:08:56 DEBUG }, 20:08:56 DEBUG "value": [ 20:08:56 DEBUG 1713640136, 20:08:56 DEBUG "0" 20:08:56 DEBUG ] 20:08:56 DEBUG } 20:08:56 DEBUG ] 20:08:56 DEBUG } 20:08:56 DEBUG } 20:08:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640137" 20:08:57 INFO [http_cmd]: http status code OK 20:08:57 DEBUG --- status code --- 20:08:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:57 DEBUG --- http response --- 20:08:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:57 DEBUG ----- output ----- 20:08:57 DEBUG { 20:08:57 DEBUG "status": "success", 20:08:57 DEBUG "data": { 20:08:57 DEBUG "resultType": "vector", 20:08:57 DEBUG "result": [ 20:08:57 DEBUG { 20:08:57 DEBUG "metric": { 20:08:57 DEBUG "pod": "ds-idrepo-0" 20:08:57 DEBUG }, 20:08:57 DEBUG "value": [ 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG [...] 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG "pod": "ds-idrepo-2" 20:08:57 DEBUG }, 20:08:57 DEBUG "value": [ 20:08:57 DEBUG 1713640137, 20:08:57 DEBUG "0" 20:08:57 DEBUG ] 20:08:57 DEBUG } 20:08:57 DEBUG ] 20:08:57 DEBUG } 20:08:57 DEBUG } 20:08:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640137" 20:08:57 INFO [http_cmd]: http status code OK 20:08:57 DEBUG --- status code --- 20:08:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:57 DEBUG --- http response --- 20:08:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:08:57 DEBUG ----- output ----- 20:08:57 DEBUG { 20:08:57 DEBUG "status": "success", 20:08:57 DEBUG "data": { 20:08:57 DEBUG "resultType": "vector", 20:08:57 DEBUG "result": [ 20:08:57 DEBUG { 20:08:57 DEBUG "metric": { 20:08:57 DEBUG "pod": "ds-cts-0" 20:08:57 DEBUG }, 20:08:57 DEBUG "value": [ 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG [...] 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG "pod": "ds-cts-2" 20:08:57 DEBUG }, 20:08:57 DEBUG "value": [ 20:08:57 DEBUG 1713640137, 20:08:57 DEBUG "0" 20:08:57 DEBUG ] 20:08:57 DEBUG } 20:08:57 DEBUG ] 20:08:57 DEBUG } 20:08:57 DEBUG } 20:08:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640137" 20:08:57 INFO 20:08:57 INFO [loop_until]: kubectl --namespace=xlou top node 20:08:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:08:57 INFO [http_cmd]: http status code OK 20:08:57 DEBUG --- status code --- 20:08:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:57 DEBUG --- http response --- 20:08:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:08:57 DEBUG ----- output ----- 20:08:57 DEBUG { 20:08:57 DEBUG "status": "success", 20:08:57 DEBUG "data": { 20:08:57 DEBUG "resultType": "vector", 20:08:57 DEBUG "result": [ 20:08:57 DEBUG { 20:08:57 DEBUG "metric": { 20:08:57 DEBUG "pod": "ds-idrepo-0" 20:08:57 DEBUG }, 20:08:57 DEBUG "value": [ 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG [...] 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG "pod": "ds-cts-2" 20:08:57 DEBUG }, 20:08:57 DEBUG "value": [ 20:08:57 DEBUG 1713640137, 20:08:57 DEBUG "0" 20:08:57 DEBUG ] 20:08:57 DEBUG } 20:08:57 DEBUG ] 20:08:57 DEBUG } 20:08:57 DEBUG } 20:08:57 INFO [loop_until]: OK (rc = 0) 20:08:57 DEBUG --- stdout --- 20:08:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6212Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8182m 27% 7646Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18260m 61% 11375Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3344Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19705m 66% 14153Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5203Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18479m 61% 11957Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15667m 52% 2969Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20931m 70% 10904Mi 9% gke-xlou-cdm-ds-0f601751-j488 8729m 29% 7237Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18924m 63% 15784Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8378m 28% 7851Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19569m 65% 13616Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9741m 61% 6394Mi 10% 20:08:57 DEBUG --- stderr --- 20:08:57 DEBUG 20:08:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640137" 20:08:57 INFO [http_cmd]: http status code OK 20:08:57 DEBUG --- status code --- 20:08:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:57 DEBUG --- http response --- 20:08:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:57 DEBUG ----- output ----- 20:08:57 DEBUG { 20:08:57 DEBUG "status": "success", 20:08:57 DEBUG "data": { 20:08:57 DEBUG "resultType": "vector", 20:08:57 DEBUG "result": [ 20:08:57 DEBUG { 20:08:57 DEBUG "metric": { 20:08:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:57 DEBUG }, 20:08:57 DEBUG "value": [ 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG [...] 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG 20:08:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:57 DEBUG }, 20:08:57 DEBUG "value": [ 20:08:57 DEBUG 1713640137, 20:08:57 DEBUG "510725.5" 20:08:57 DEBUG ] 20:08:57 DEBUG } 20:08:57 DEBUG ] 20:08:57 DEBUG } 20:08:57 DEBUG } 20:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640138" 20:08:58 INFO [http_cmd]: http status code OK 20:08:58 DEBUG --- status code --- 20:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:58 DEBUG --- http response --- 20:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:58 DEBUG ----- output ----- 20:08:58 DEBUG { 20:08:58 DEBUG "status": "success", 20:08:58 DEBUG "data": { 20:08:58 DEBUG "resultType": "vector", 20:08:58 DEBUG "result": [ 20:08:58 DEBUG { 20:08:58 DEBUG "metric": { 20:08:58 DEBUG "instance": "10.142.0.107:9100" 20:08:58 DEBUG }, 20:08:58 DEBUG "value": [ 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG [...] 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG "instance": "10.142.0.43:9100" 20:08:58 DEBUG }, 20:08:58 DEBUG "value": [ 20:08:58 DEBUG 1713640138, 20:08:58 DEBUG "0.27939999999989784" 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG } 20:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640138" 20:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640138" 20:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640138" 20:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640138" 20:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640138" 20:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640138" 20:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640138" 20:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640138" 20:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640138" 20:08:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640138" 20:08:58 INFO [http_cmd]: http status code OK 20:08:58 INFO [http_cmd]: http status code OK 20:08:58 INFO [http_cmd]: http status code OK 20:08:58 INFO [http_cmd]: http status code OK 20:08:58 DEBUG --- status code --- 20:08:58 DEBUG --- status code --- 20:08:58 DEBUG --- status code --- 20:08:58 INFO [http_cmd]: http status code OK 20:08:58 DEBUG --- status code --- 20:08:58 INFO [http_cmd]: http status code OK 20:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:58 INFO [http_cmd]: http status code OK 20:08:58 INFO [http_cmd]: http status code OK 20:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:58 DEBUG --- status code --- 20:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:58 DEBUG --- status code --- 20:08:58 DEBUG --- http response --- 20:08:58 DEBUG --- http response --- 20:08:58 DEBUG --- status code --- 20:08:58 DEBUG --- status code --- 20:08:58 DEBUG --- http response --- 20:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:58 DEBUG --- http response --- 20:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:58 DEBUG ----- output ----- 20:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:08:58 DEBUG --- http response --- 20:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:08:58 DEBUG --- http response --- 20:08:58 DEBUG { 20:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:58 INFO [http_cmd]: http status code OK 20:08:58 DEBUG --- http response --- 20:08:58 DEBUG --- http response --- 20:08:58 DEBUG ----- output ----- 20:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:58 DEBUG ----- output ----- 20:08:58 INFO [http_cmd]: http status code OK 20:08:58 DEBUG "status": "success", 20:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:08:58 DEBUG ----- output ----- 20:08:58 DEBUG { 20:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:58 DEBUG { 20:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:08:58 DEBUG ----- output ----- 20:08:58 DEBUG { 20:08:58 DEBUG --- status code --- 20:08:58 DEBUG "data": { 20:08:58 DEBUG ----- output ----- 20:08:58 DEBUG "status": "success", 20:08:58 DEBUG --- status code --- 20:08:58 DEBUG ----- output ----- 20:08:58 DEBUG "status": "success", 20:08:58 DEBUG ----- output ----- 20:08:58 DEBUG { 20:08:58 DEBUG "status": "success", 20:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:58 DEBUG "resultType": "vector", 20:08:58 DEBUG { 20:08:58 DEBUG "data": { 20:08:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:08:58 DEBUG { 20:08:58 DEBUG "data": { 20:08:58 DEBUG { 20:08:58 DEBUG "status": "success", 20:08:58 DEBUG "data": { 20:08:58 DEBUG --- http response --- 20:08:58 DEBUG "result": [ 20:08:58 DEBUG "status": "success", 20:08:58 DEBUG "resultType": "vector", 20:08:58 DEBUG --- http response --- 20:08:58 DEBUG "status": "success", 20:08:58 DEBUG "resultType": "vector", 20:08:58 DEBUG "status": "success", 20:08:58 DEBUG "data": { 20:08:58 DEBUG "resultType": "vector", 20:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:58 DEBUG { 20:08:58 DEBUG "data": { 20:08:58 DEBUG "result": [ 20:08:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:08:58 DEBUG "data": { 20:08:58 DEBUG "result": [ 20:08:58 DEBUG "data": { 20:08:58 DEBUG "resultType": "vector", 20:08:58 DEBUG "result": [ 20:08:58 DEBUG ----- output ----- 20:08:58 DEBUG "metric": { 20:08:58 DEBUG "resultType": "vector", 20:08:58 DEBUG { 20:08:58 DEBUG ----- output ----- 20:08:58 DEBUG "resultType": "vector", 20:08:58 DEBUG { 20:08:58 DEBUG "resultType": "vector", 20:08:58 DEBUG "result": [ 20:08:58 DEBUG { 20:08:58 DEBUG "metric": { 20:08:58 DEBUG "instance": "10.142.0.107:9100" 20:08:58 DEBUG "result": [ 20:08:58 DEBUG "metric": { 20:08:58 DEBUG { 20:08:58 DEBUG "result": [ 20:08:58 DEBUG "metric": { 20:08:58 DEBUG "result": [ 20:08:58 DEBUG { 20:08:58 DEBUG { 20:08:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:08:58 DEBUG }, 20:08:58 DEBUG { 20:08:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:08:58 DEBUG "status": "success", 20:08:58 DEBUG { 20:08:58 DEBUG "pod": "ds-cts-0" 20:08:58 DEBUG { 20:08:58 DEBUG "metric": { 20:08:58 DEBUG "status": "success", 20:08:58 DEBUG }, 20:08:58 DEBUG "value": [ 20:08:58 DEBUG "metric": { 20:08:58 DEBUG }, 20:08:58 DEBUG "data": { 20:08:58 DEBUG "metric": { 20:08:58 DEBUG }, 20:08:58 DEBUG "metric": { 20:08:58 DEBUG "instance": "10.142.0.107:9100" 20:08:58 DEBUG "data": { 20:08:58 DEBUG "value": [ 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG "value": [ 20:08:58 DEBUG "resultType": "vector", 20:08:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:58 DEBUG "value": [ 20:08:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:08:58 DEBUG }, 20:08:58 DEBUG "resultType": "vector", 20:08:58 DEBUG "pod": "ds-cts-0" 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG "result": [ 20:08:58 DEBUG }, 20:08:58 DEBUG 20:08:58 DEBUG }, 20:08:58 DEBUG "value": [ 20:08:58 DEBUG "result": [ 20:08:58 DEBUG }, 20:08:58 DEBUG [...] 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG { 20:08:58 DEBUG "value": [ 20:08:58 DEBUG 20:08:58 DEBUG "value": [ 20:08:58 DEBUG 20:08:58 DEBUG { 20:08:58 DEBUG "value": [ 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG "metric": { 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG "metric": { 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG [...] 20:08:58 DEBUG [...] 20:08:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:08:58 DEBUG 20:08:58 DEBUG [...] 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG "instance": "10.142.0.107:9100" 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG }, 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG [...] 20:08:58 DEBUG }, 20:08:58 DEBUG 20:08:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG "value": [ 20:08:58 DEBUG [...] 20:08:58 DEBUG 20:08:58 DEBUG [...] 20:08:58 DEBUG 20:08:58 DEBUG "value": [ 20:08:58 DEBUG [...] 20:08:58 DEBUG }, 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG "value": [ 20:08:58 DEBUG "instance": "10.142.0.43:9100" 20:08:58 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG "pod": "ds-cts-2" 20:08:58 DEBUG }, 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 1713640138, 20:08:58 DEBUG }, 20:08:58 DEBUG }, 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG "value": [ 20:08:58 DEBUG "instance": "10.142.0.43:9100" 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG "0" 20:08:58 DEBUG "value": [ 20:08:58 DEBUG "value": [ 20:08:58 DEBUG [...] 20:08:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:58 DEBUG 1713640138, 20:08:58 DEBUG }, 20:08:58 DEBUG 20:08:58 DEBUG [...] 20:08:58 DEBUG "pod": "ds-cts-2" 20:08:58 DEBUG ] 20:08:58 DEBUG 1713640138, 20:08:58 DEBUG 1713640138, 20:08:58 DEBUG 20:08:58 DEBUG }, 20:08:58 DEBUG "0.016893626666666665" 20:08:58 DEBUG "value": [ 20:08:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:08:58 DEBUG }, 20:08:58 DEBUG }, 20:08:58 DEBUG } 20:08:58 DEBUG "85770202.76666667" 20:08:58 DEBUG ] 20:08:58 DEBUG 20:08:58 DEBUG 20:08:58 DEBUG ] 20:08:58 DEBUG 1713640138, 20:08:58 DEBUG "value": [ 20:08:58 DEBUG "value": [ 20:08:58 DEBUG 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG "0" 20:08:58 DEBUG "value": [ 20:08:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:08:58 DEBUG }, 20:08:58 DEBUG "85765150.4" 20:08:58 DEBUG 1713640138, 20:08:58 DEBUG "0" 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG 1713640138, 20:08:58 DEBUG "value": [ 20:08:58 DEBUG } 20:08:58 DEBUG ] 20:08:58 DEBUG 20:08:58 DEBUG ] 20:08:58 DEBUG 1713640138, 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG } 20:08:58 DEBUG "0.157" 20:08:58 DEBUG 1713640138, 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG 20:08:58 DEBUG } 20:08:58 DEBUG "10.6" 20:08:58 DEBUG } 20:08:58 DEBUG } 20:08:58 DEBUG ] 20:08:58 DEBUG "952123392" 20:08:58 DEBUG } 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG } 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG } 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG } 20:08:58 DEBUG "instance": "10.142.0.43:9100" 20:08:58 DEBUG ] 20:08:58 DEBUG ] 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG }, 20:08:58 DEBUG } 20:08:58 DEBUG } 20:08:58 DEBUG } 20:08:58 DEBUG ] 20:08:58 DEBUG "value": [ 20:08:58 DEBUG 1713640138, 20:08:58 DEBUG } 20:08:58 DEBUG } 20:08:58 DEBUG } 20:08:58 DEBUG } 20:08:58 DEBUG "0.09033333333336106" 20:08:58 DEBUG } 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG ] 20:08:58 DEBUG } 20:08:58 DEBUG } 20:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640193" 20:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640193" 20:09:53 INFO [http_cmd]: http status code OK 20:09:53 DEBUG --- status code --- 20:09:53 ERROR [http_cmd]: ERROR 20:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:53 ERROR --- status code --- 20:09:53 DEBUG --- http response --- 20:09:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:09:53 ERROR --- http response --- 20:09:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:09:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:09:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:09:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640194" 20:09:54 INFO [http_cmd]: http status code OK 20:09:54 DEBUG --- status code --- 20:09:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:54 DEBUG --- http response --- 20:09:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:54 DEBUG ----- output ----- 20:09:54 DEBUG { 20:09:54 DEBUG "status": "success", 20:09:54 DEBUG "data": { 20:09:54 DEBUG "resultType": "vector", 20:09:54 DEBUG "result": [ 20:09:54 DEBUG { 20:09:54 DEBUG "metric": { 20:09:54 DEBUG "pod": "ds-idrepo-0" 20:09:54 DEBUG }, 20:09:54 DEBUG "value": [ 20:09:54 DEBUG 20:09:54 DEBUG 20:09:54 DEBUG 20:09:54 DEBUG [...] 20:09:54 DEBUG 20:09:54 DEBUG 20:09:54 DEBUG 20:09:54 DEBUG "pod": "ds-idrepo-2" 20:09:54 DEBUG }, 20:09:54 DEBUG "value": [ 20:09:54 DEBUG 1713640194, 20:09:54 DEBUG "0" 20:09:54 DEBUG ] 20:09:54 DEBUG } 20:09:54 DEBUG ] 20:09:54 DEBUG } 20:09:54 DEBUG } 20:09:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640194" 20:09:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640194" 20:09:54 INFO [http_cmd]: http status code OK 20:09:54 DEBUG --- status code --- 20:09:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:54 DEBUG --- http response --- 20:09:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:54 DEBUG ----- output ----- 20:09:54 DEBUG { 20:09:54 DEBUG "status": "success", 20:09:54 DEBUG "data": { 20:09:54 DEBUG "resultType": "vector", 20:09:54 DEBUG "result": [ 20:09:54 DEBUG { 20:09:54 DEBUG "metric": { 20:09:54 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:54 DEBUG }, 20:09:54 DEBUG "value": [ 20:09:54 DEBUG 20:09:54 DEBUG 20:09:54 DEBUG 20:09:54 DEBUG [...] 20:09:54 DEBUG 20:09:54 DEBUG 20:09:54 DEBUG 20:09:54 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:54 DEBUG }, 20:09:54 DEBUG "value": [ 20:09:54 DEBUG 1713640194, 20:09:54 DEBUG "2834.4333333333334" 20:09:54 DEBUG ] 20:09:54 DEBUG } 20:09:54 DEBUG ] 20:09:54 DEBUG } 20:09:54 DEBUG } 20:09:54 INFO [http_cmd]: http status code OK 20:09:54 DEBUG --- status code --- 20:09:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:54 DEBUG --- http response --- 20:09:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:54 DEBUG ----- output ----- 20:09:54 DEBUG { 20:09:54 DEBUG "status": "success", 20:09:54 DEBUG "data": { 20:09:54 DEBUG "resultType": "vector", 20:09:54 DEBUG "result": [ 20:09:54 DEBUG { 20:09:54 DEBUG "metric": { 20:09:54 DEBUG "pod": "ds-idrepo-0" 20:09:54 DEBUG }, 20:09:54 DEBUG "value": [ 20:09:54 DEBUG 20:09:54 DEBUG 20:09:54 DEBUG 20:09:54 DEBUG [...] 20:09:54 DEBUG 20:09:54 DEBUG 20:09:54 DEBUG 20:09:54 DEBUG "pod": "ds-idrepo-2" 20:09:54 DEBUG }, 20:09:54 DEBUG "value": [ 20:09:54 DEBUG 1713640194, 20:09:54 DEBUG "0" 20:09:54 DEBUG ] 20:09:54 DEBUG } 20:09:54 DEBUG ] 20:09:54 DEBUG } 20:09:54 DEBUG } 20:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640195" 20:09:55 INFO [http_cmd]: http status code OK 20:09:55 DEBUG --- status code --- 20:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:55 DEBUG --- http response --- 20:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:55 DEBUG ----- output ----- 20:09:55 DEBUG { 20:09:55 DEBUG "status": "success", 20:09:55 DEBUG "data": { 20:09:55 DEBUG "resultType": "vector", 20:09:55 DEBUG "result": [ 20:09:55 DEBUG { 20:09:55 DEBUG "metric": { 20:09:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:09:55 DEBUG }, 20:09:55 DEBUG "value": [ 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG [...] 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:09:55 DEBUG }, 20:09:55 DEBUG "value": [ 20:09:55 DEBUG 1713640195, 20:09:55 DEBUG "0" 20:09:55 DEBUG ] 20:09:55 DEBUG } 20:09:55 DEBUG ] 20:09:55 DEBUG } 20:09:55 DEBUG } 20:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640195" 20:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640195" 20:09:55 INFO [http_cmd]: http status code OK 20:09:55 DEBUG --- status code --- 20:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:55 DEBUG --- http response --- 20:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:55 DEBUG ----- output ----- 20:09:55 DEBUG { 20:09:55 DEBUG "status": "success", 20:09:55 DEBUG "data": { 20:09:55 DEBUG "resultType": "vector", 20:09:55 DEBUG "result": [ 20:09:55 DEBUG { 20:09:55 DEBUG "metric": { 20:09:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:55 DEBUG }, 20:09:55 DEBUG "value": [ 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG [...] 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:55 DEBUG }, 20:09:55 DEBUG "value": [ 20:09:55 DEBUG 1713640195, 20:09:55 DEBUG "2840.733333333333" 20:09:55 DEBUG ] 20:09:55 DEBUG } 20:09:55 DEBUG ] 20:09:55 DEBUG } 20:09:55 DEBUG } 20:09:55 INFO [http_cmd]: http status code OK 20:09:55 DEBUG --- status code --- 20:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:55 DEBUG --- http response --- 20:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:55 DEBUG ----- output ----- 20:09:55 DEBUG { 20:09:55 DEBUG "status": "success", 20:09:55 DEBUG "data": { 20:09:55 DEBUG "resultType": "vector", 20:09:55 DEBUG "result": [ 20:09:55 DEBUG { 20:09:55 DEBUG "metric": { 20:09:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:09:55 DEBUG }, 20:09:55 DEBUG "value": [ 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG [...] 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:09:55 DEBUG }, 20:09:55 DEBUG "value": [ 20:09:55 DEBUG 1713640195, 20:09:55 DEBUG "9.600043418395295" 20:09:55 DEBUG ] 20:09:55 DEBUG } 20:09:55 DEBUG ] 20:09:55 DEBUG } 20:09:55 DEBUG } 20:09:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640195" 20:09:55 INFO [http_cmd]: http status code OK 20:09:55 DEBUG --- status code --- 20:09:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:55 DEBUG --- http response --- 20:09:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:55 DEBUG ----- output ----- 20:09:55 DEBUG { 20:09:55 DEBUG "status": "success", 20:09:55 DEBUG "data": { 20:09:55 DEBUG "resultType": "vector", 20:09:55 DEBUG "result": [ 20:09:55 DEBUG { 20:09:55 DEBUG "metric": { 20:09:55 DEBUG "instance": "10.142.0.107:9100" 20:09:55 DEBUG }, 20:09:55 DEBUG "value": [ 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG [...] 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG 20:09:55 DEBUG "instance": "10.142.0.43:9100" 20:09:55 DEBUG }, 20:09:55 DEBUG "value": [ 20:09:55 DEBUG 1713640195, 20:09:55 DEBUG "3684954543104" 20:09:55 DEBUG ] 20:09:55 DEBUG } 20:09:55 DEBUG ] 20:09:55 DEBUG } 20:09:55 DEBUG } 20:09:56 INFO 20:09:56 INFO [loop_until]: kubectl --namespace=xlou top pods 20:09:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:09:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640196" 20:09:56 INFO [http_cmd]: http status code OK 20:09:56 DEBUG --- status code --- 20:09:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:56 DEBUG --- http response --- 20:09:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:56 DEBUG ----- output ----- 20:09:56 DEBUG { 20:09:56 DEBUG "status": "success", 20:09:56 DEBUG "data": { 20:09:56 DEBUG "resultType": "vector", 20:09:56 DEBUG "result": [ 20:09:56 DEBUG { 20:09:56 DEBUG "metric": { 20:09:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:56 DEBUG }, 20:09:56 DEBUG "value": [ 20:09:56 DEBUG 20:09:56 DEBUG 20:09:56 DEBUG 20:09:56 DEBUG [...] 20:09:56 DEBUG 20:09:56 DEBUG 20:09:56 DEBUG 20:09:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:56 DEBUG }, 20:09:56 DEBUG "value": [ 20:09:56 DEBUG 1713640196, 20:09:56 DEBUG "2836.866666666667" 20:09:56 DEBUG ] 20:09:56 DEBUG } 20:09:56 DEBUG ] 20:09:56 DEBUG } 20:09:56 DEBUG } 20:09:56 INFO [loop_until]: OK (rc = 0) 20:09:56 DEBUG --- stdout --- 20:09:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18509m 8658Mi am-767f6b67b5-qcqz2 18337m 9067Mi am-767f6b67b5-vbltz 20027m 8672Mi ds-cts-0 19940m 10969Mi ds-cts-1 18469m 10776Mi ds-cts-2 20054m 10955Mi ds-idrepo-0 8114m 3894Mi ds-idrepo-1 7922m 3882Mi ds-idrepo-2 7885m 3828Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9592m 814Mi 20:09:56 DEBUG --- stderr --- 20:09:56 DEBUG 20:09:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640196" 20:09:56 INFO [http_cmd]: http status code OK 20:09:56 DEBUG --- status code --- 20:09:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:56 DEBUG --- http response --- 20:09:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:56 DEBUG ----- output ----- 20:09:56 DEBUG { 20:09:56 DEBUG "status": "success", 20:09:56 DEBUG "data": { 20:09:56 DEBUG "resultType": "vector", 20:09:56 DEBUG "result": [ 20:09:56 DEBUG { 20:09:56 DEBUG "metric": { 20:09:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:56 DEBUG }, 20:09:56 DEBUG "value": [ 20:09:56 DEBUG 20:09:56 DEBUG 20:09:56 DEBUG 20:09:56 DEBUG [...] 20:09:56 DEBUG 20:09:56 DEBUG 20:09:56 DEBUG 20:09:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:56 DEBUG }, 20:09:56 DEBUG "value": [ 20:09:56 DEBUG 1713640196, 20:09:56 DEBUG "2836.8" 20:09:56 DEBUG ] 20:09:56 DEBUG } 20:09:56 DEBUG ] 20:09:56 DEBUG } 20:09:56 DEBUG } 20:09:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640196" 20:09:56 INFO [http_cmd]: http status code OK 20:09:56 DEBUG --- status code --- 20:09:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:56 DEBUG --- http response --- 20:09:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:09:56 DEBUG ----- output ----- 20:09:56 DEBUG { 20:09:56 DEBUG "status": "success", 20:09:56 DEBUG "data": { 20:09:56 DEBUG "resultType": "vector", 20:09:56 DEBUG "result": [ 20:09:56 DEBUG { 20:09:56 DEBUG "metric": { 20:09:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:09:56 DEBUG }, 20:09:56 DEBUG "value": [ 20:09:56 DEBUG 20:09:56 DEBUG 20:09:56 DEBUG 20:09:56 DEBUG [...] 20:09:56 DEBUG 20:09:56 DEBUG 20:09:56 DEBUG 20:09:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:09:56 DEBUG }, 20:09:56 DEBUG "value": [ 20:09:56 DEBUG 1713640196, 20:09:56 DEBUG "0.00003509581263306191" 20:09:56 DEBUG ] 20:09:56 DEBUG } 20:09:56 DEBUG ] 20:09:56 DEBUG } 20:09:56 DEBUG } 20:09:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640197" 20:09:57 INFO [http_cmd]: http status code OK 20:09:57 DEBUG --- status code --- 20:09:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:57 DEBUG --- http response --- 20:09:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:57 DEBUG ----- output ----- 20:09:57 DEBUG { 20:09:57 DEBUG "status": "success", 20:09:57 DEBUG "data": { 20:09:57 DEBUG "resultType": "vector", 20:09:57 DEBUG "result": [ 20:09:57 DEBUG { 20:09:57 DEBUG "metric": { 20:09:57 DEBUG "pod": "ds-idrepo-0" 20:09:57 DEBUG }, 20:09:57 DEBUG "value": [ 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG [...] 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG "pod": "ds-idrepo-2" 20:09:57 DEBUG }, 20:09:57 DEBUG "value": [ 20:09:57 DEBUG 1713640197, 20:09:57 DEBUG "0" 20:09:57 DEBUG ] 20:09:57 DEBUG } 20:09:57 DEBUG ] 20:09:57 DEBUG } 20:09:57 DEBUG } 20:09:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640197" 20:09:57 INFO [http_cmd]: http status code OK 20:09:57 DEBUG --- status code --- 20:09:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:57 DEBUG --- http response --- 20:09:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:09:57 DEBUG ----- output ----- 20:09:57 DEBUG { 20:09:57 DEBUG "status": "success", 20:09:57 DEBUG "data": { 20:09:57 DEBUG "resultType": "vector", 20:09:57 DEBUG "result": [ 20:09:57 DEBUG { 20:09:57 DEBUG "metric": { 20:09:57 DEBUG "pod": "ds-cts-0" 20:09:57 DEBUG }, 20:09:57 DEBUG "value": [ 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG [...] 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG "pod": "ds-cts-2" 20:09:57 DEBUG }, 20:09:57 DEBUG "value": [ 20:09:57 DEBUG 1713640197, 20:09:57 DEBUG "0" 20:09:57 DEBUG ] 20:09:57 DEBUG } 20:09:57 DEBUG ] 20:09:57 DEBUG } 20:09:57 DEBUG } 20:09:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640197" 20:09:57 INFO [http_cmd]: http status code OK 20:09:57 DEBUG --- status code --- 20:09:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:57 DEBUG --- http response --- 20:09:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:09:57 DEBUG ----- output ----- 20:09:57 DEBUG { 20:09:57 DEBUG "status": "success", 20:09:57 DEBUG "data": { 20:09:57 DEBUG "resultType": "vector", 20:09:57 DEBUG "result": [ 20:09:57 DEBUG { 20:09:57 DEBUG "metric": { 20:09:57 DEBUG "pod": "ds-idrepo-0" 20:09:57 DEBUG }, 20:09:57 DEBUG "value": [ 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG [...] 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG "pod": "ds-cts-2" 20:09:57 DEBUG }, 20:09:57 DEBUG "value": [ 20:09:57 DEBUG 1713640197, 20:09:57 DEBUG "0" 20:09:57 DEBUG ] 20:09:57 DEBUG } 20:09:57 DEBUG ] 20:09:57 DEBUG } 20:09:57 DEBUG } 20:09:57 INFO 20:09:57 INFO [loop_until]: kubectl --namespace=xlou top node 20:09:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:09:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640197" 20:09:57 INFO [http_cmd]: http status code OK 20:09:57 DEBUG --- status code --- 20:09:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:57 DEBUG --- http response --- 20:09:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:57 DEBUG ----- output ----- 20:09:57 DEBUG { 20:09:57 DEBUG "status": "success", 20:09:57 DEBUG "data": { 20:09:57 DEBUG "resultType": "vector", 20:09:57 DEBUG "result": [ 20:09:57 DEBUG { 20:09:57 DEBUG "metric": { 20:09:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:57 DEBUG }, 20:09:57 DEBUG "value": [ 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG [...] 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG 20:09:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:57 DEBUG }, 20:09:57 DEBUG "value": [ 20:09:57 DEBUG 1713640197, 20:09:57 DEBUG "510831.5" 20:09:57 DEBUG ] 20:09:57 DEBUG } 20:09:57 DEBUG ] 20:09:57 DEBUG } 20:09:57 DEBUG } 20:09:57 INFO [loop_until]: OK (rc = 0) 20:09:57 DEBUG --- stdout --- 20:09:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6205Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7933m 26% 7641Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18275m 61% 11382Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3341Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20137m 67% 14163Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5191Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18543m 62% 11947Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15237m 51% 2950Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20861m 69% 10892Mi 9% gke-xlou-cdm-ds-0f601751-j488 8745m 29% 7236Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18559m 62% 15820Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8649m 28% 7856Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20217m 67% 13612Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9797m 61% 6396Mi 10% 20:09:57 DEBUG --- stderr --- 20:09:57 DEBUG 20:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640198" 20:09:58 INFO [http_cmd]: http status code OK 20:09:58 DEBUG --- status code --- 20:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:58 DEBUG --- http response --- 20:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:58 DEBUG ----- output ----- 20:09:58 DEBUG { 20:09:58 DEBUG "status": "success", 20:09:58 DEBUG "data": { 20:09:58 DEBUG "resultType": "vector", 20:09:58 DEBUG "result": [ 20:09:58 DEBUG { 20:09:58 DEBUG "metric": { 20:09:58 DEBUG "instance": "10.142.0.107:9100" 20:09:58 DEBUG }, 20:09:58 DEBUG "value": [ 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG [...] 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG "instance": "10.142.0.43:9100" 20:09:58 DEBUG }, 20:09:58 DEBUG "value": [ 20:09:58 DEBUG 1713640198, 20:09:58 DEBUG "0.16719999999999968" 20:09:58 DEBUG ] 20:09:58 DEBUG } 20:09:58 DEBUG ] 20:09:58 DEBUG } 20:09:58 DEBUG } 20:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640198" 20:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640198" 20:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640198" 20:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640198" 20:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640198" 20:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640198" 20:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640198" 20:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640198" 20:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640198" 20:09:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640198" 20:09:58 INFO [http_cmd]: http status code OK 20:09:58 DEBUG --- status code --- 20:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:58 DEBUG --- http response --- 20:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:58 DEBUG ----- output ----- 20:09:58 INFO [http_cmd]: http status code OK 20:09:58 INFO [http_cmd]: http status code OK 20:09:58 INFO [http_cmd]: http status code OK 20:09:58 DEBUG { 20:09:58 INFO [http_cmd]: http status code OK 20:09:58 INFO [http_cmd]: http status code OK 20:09:58 DEBUG --- status code --- 20:09:58 DEBUG --- status code --- 20:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:58 DEBUG --- status code --- 20:09:58 DEBUG "status": "success", 20:09:58 INFO [http_cmd]: http status code OK 20:09:58 DEBUG --- status code --- 20:09:58 DEBUG --- status code --- 20:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:58 INFO [http_cmd]: http status code OK 20:09:58 DEBUG --- http response --- 20:09:58 INFO [http_cmd]: http status code OK 20:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:58 DEBUG "data": { 20:09:58 INFO [http_cmd]: http status code OK 20:09:58 DEBUG --- status code --- 20:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:58 DEBUG --- http response --- 20:09:58 DEBUG --- status code --- 20:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:58 DEBUG --- status code --- 20:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:58 DEBUG "resultType": "vector", 20:09:58 DEBUG --- status code --- 20:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:58 DEBUG --- http response --- 20:09:58 DEBUG --- http response --- 20:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:58 DEBUG ----- output ----- 20:09:58 DEBUG --- http response --- 20:09:58 DEBUG --- http response --- 20:09:58 DEBUG "result": [ 20:09:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:09:58 DEBUG --- http response --- 20:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:09:58 DEBUG ----- output ----- 20:09:58 DEBUG --- http response --- 20:09:58 DEBUG { 20:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:58 DEBUG { 20:09:58 DEBUG --- http response --- 20:09:58 DEBUG ----- output ----- 20:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:58 DEBUG ----- output ----- 20:09:58 DEBUG { 20:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:09:58 DEBUG "status": "success", 20:09:58 DEBUG ----- output ----- 20:09:58 DEBUG ----- output ----- 20:09:58 DEBUG "metric": { 20:09:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:09:58 DEBUG { 20:09:58 DEBUG ----- output ----- 20:09:58 DEBUG { 20:09:58 DEBUG "status": "success", 20:09:58 DEBUG ----- output ----- 20:09:58 DEBUG "data": { 20:09:58 DEBUG { 20:09:58 DEBUG { 20:09:58 DEBUG "status": "success", 20:09:58 DEBUG ----- output ----- 20:09:58 DEBUG "status": "success", 20:09:58 DEBUG { 20:09:58 DEBUG "status": "success", 20:09:58 DEBUG "data": { 20:09:58 DEBUG { 20:09:58 DEBUG "resultType": "vector", 20:09:58 DEBUG "status": "success", 20:09:58 DEBUG "data": { 20:09:58 DEBUG "resultType": "vector", 20:09:58 DEBUG "result": [ 20:09:58 DEBUG "data": { 20:09:58 DEBUG "status": "success", 20:09:58 DEBUG "data": { 20:09:58 DEBUG "resultType": "vector", 20:09:58 DEBUG "status": "success", 20:09:58 DEBUG "result": [ 20:09:58 DEBUG { 20:09:58 DEBUG { 20:09:58 DEBUG "data": { 20:09:58 DEBUG "instance": "10.142.0.107:9100" 20:09:58 DEBUG "resultType": "vector", 20:09:58 DEBUG "resultType": "vector", 20:09:58 DEBUG "data": { 20:09:58 DEBUG "result": [ 20:09:58 DEBUG "data": { 20:09:58 DEBUG { 20:09:58 DEBUG "metric": { 20:09:58 DEBUG "status": "success", 20:09:58 DEBUG "resultType": "vector", 20:09:58 DEBUG }, 20:09:58 DEBUG "result": [ 20:09:58 DEBUG "result": [ 20:09:58 DEBUG "resultType": "vector", 20:09:58 DEBUG { 20:09:58 DEBUG "resultType": "vector", 20:09:58 DEBUG "metric": { 20:09:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:09:58 DEBUG "data": { 20:09:58 DEBUG "result": [ 20:09:58 DEBUG "value": [ 20:09:58 DEBUG 20:09:58 DEBUG { 20:09:58 DEBUG "result": [ 20:09:58 DEBUG "metric": { 20:09:58 DEBUG "result": [ 20:09:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:58 DEBUG }, 20:09:58 DEBUG "resultType": "vector", 20:09:58 DEBUG { 20:09:58 DEBUG { 20:09:58 DEBUG 20:09:58 DEBUG "metric": { 20:09:58 DEBUG { 20:09:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:09:58 DEBUG { 20:09:58 DEBUG }, 20:09:58 DEBUG "value": [ 20:09:58 DEBUG "result": [ 20:09:58 DEBUG "metric": { 20:09:58 DEBUG "metric": { 20:09:58 DEBUG 20:09:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:09:58 DEBUG "metric": { 20:09:58 DEBUG }, 20:09:58 DEBUG "metric": { 20:09:58 DEBUG "value": [ 20:09:58 DEBUG 20:09:58 DEBUG { 20:09:58 DEBUG "instance": "10.142.0.107:9100" 20:09:58 DEBUG "pod": "ds-cts-0" 20:09:58 DEBUG [...] 20:09:58 DEBUG }, 20:09:58 DEBUG "pod": "ds-cts-0" 20:09:58 DEBUG "value": [ 20:09:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG }, 20:09:58 DEBUG }, 20:09:58 DEBUG 20:09:58 DEBUG "value": [ 20:09:58 DEBUG }, 20:09:58 DEBUG 20:09:58 DEBUG }, 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG [...] 20:09:58 DEBUG "value": [ 20:09:58 DEBUG "value": [ 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG "instance": "10.142.0.43:9100" 20:09:58 DEBUG 20:09:58 DEBUG "value": [ 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG "value": [ 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG [...] 20:09:58 DEBUG [...] 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG }, 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG [...] 20:09:58 DEBUG 20:09:58 DEBUG "metric": { 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG [...] 20:09:58 DEBUG [...] 20:09:58 DEBUG [...] 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG "instance": "10.142.0.107:9100" 20:09:58 DEBUG 20:09:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:09:58 DEBUG 20:09:58 DEBUG "value": [ 20:09:58 DEBUG 1713640198, 20:09:58 DEBUG "86014998.6" 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG }, 20:09:58 DEBUG 20:09:58 DEBUG }, 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG ] 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG "pod": "ds-cts-2" 20:09:58 DEBUG "value": [ 20:09:58 DEBUG 20:09:58 DEBUG "value": [ 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG } 20:09:58 DEBUG 20:09:58 DEBUG [...] 20:09:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:58 DEBUG }, 20:09:58 DEBUG "value": [ 20:09:58 DEBUG 20:09:58 DEBUG 1713640198, 20:09:58 DEBUG "instance": "10.142.0.43:9100" 20:09:58 DEBUG }, 20:09:58 DEBUG ] 20:09:58 DEBUG 20:09:58 DEBUG 20:09:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:09:58 DEBUG 1713640198, 20:09:58 DEBUG 20:09:58 DEBUG "0" 20:09:58 DEBUG }, 20:09:58 DEBUG 20:09:58 DEBUG "value": [ 20:09:58 DEBUG } 20:09:58 DEBUG "pod": "ds-cts-2" 20:09:58 DEBUG 20:09:58 DEBUG }, 20:09:58 DEBUG "0" 20:09:58 DEBUG [...] 20:09:58 DEBUG ] 20:09:58 DEBUG "value": [ 20:09:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:09:58 DEBUG 1713640198, 20:09:58 DEBUG } 20:09:58 DEBUG }, 20:09:58 DEBUG 20:09:58 DEBUG "value": [ 20:09:58 DEBUG ] 20:09:58 DEBUG 20:09:58 DEBUG } 20:09:58 DEBUG 1713640198, 20:09:58 DEBUG "86009804.26666667" 20:09:58 DEBUG "29.466666666666665" 20:09:58 DEBUG "value": [ 20:09:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:09:58 DEBUG 1713640198, 20:09:58 DEBUG "0.157" 20:09:58 DEBUG 20:09:58 DEBUG ] 20:09:58 DEBUG ] 20:09:58 DEBUG }, 20:09:58 DEBUG ] 20:09:58 DEBUG 1713640198, 20:09:58 DEBUG }, 20:09:58 DEBUG ] 20:09:58 DEBUG } 20:09:58 DEBUG 20:09:58 DEBUG } 20:09:58 DEBUG } 20:09:58 DEBUG "value": [ 20:09:58 DEBUG } 20:09:58 DEBUG "0.023800000000000005" 20:09:58 DEBUG "value": [ 20:09:58 DEBUG } 20:09:58 DEBUG ] 20:09:58 DEBUG "instance": "10.142.0.43:9100" 20:09:58 DEBUG } 20:09:58 DEBUG ] 20:09:58 DEBUG 1713640198, 20:09:58 DEBUG ] 20:09:58 DEBUG } 20:09:58 DEBUG 1713640198, 20:09:58 DEBUG "0" 20:09:58 DEBUG } 20:09:58 DEBUG }, 20:09:58 DEBUG } 20:09:58 DEBUG "955686912" 20:09:58 DEBUG } 20:09:58 DEBUG ] 20:09:58 DEBUG ] 20:09:58 DEBUG ] 20:09:58 DEBUG } 20:09:58 DEBUG "value": [ 20:09:58 DEBUG } 20:09:58 DEBUG ] 20:09:58 DEBUG } 20:09:58 DEBUG } 20:09:58 DEBUG ] 20:09:58 DEBUG 1713640198, 20:09:58 DEBUG } 20:09:58 DEBUG ] 20:09:58 DEBUG } 20:09:58 DEBUG } 20:09:58 DEBUG "0.05700000000000689" 20:09:58 DEBUG ] 20:09:58 DEBUG } 20:09:58 DEBUG } 20:09:58 DEBUG } 20:09:58 DEBUG ] 20:09:58 DEBUG } 20:09:58 DEBUG } 20:09:58 DEBUG } 20:09:58 DEBUG } 20:09:58 DEBUG ] 20:09:58 DEBUG } 20:09:58 DEBUG } 20:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640253" 20:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640253" 20:10:53 INFO [http_cmd]: http status code OK 20:10:53 DEBUG --- status code --- 20:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:53 DEBUG --- http response --- 20:10:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:10:53 ERROR [http_cmd]: ERROR 20:10:53 ERROR --- status code --- 20:10:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:10:53 ERROR --- http response --- 20:10:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:10:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640254" 20:10:54 INFO [http_cmd]: http status code OK 20:10:54 DEBUG --- status code --- 20:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:54 DEBUG --- http response --- 20:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:54 DEBUG ----- output ----- 20:10:54 DEBUG { 20:10:54 DEBUG "status": "success", 20:10:54 DEBUG "data": { 20:10:54 DEBUG "resultType": "vector", 20:10:54 DEBUG "result": [ 20:10:54 DEBUG { 20:10:54 DEBUG "metric": { 20:10:54 DEBUG "pod": "ds-idrepo-0" 20:10:54 DEBUG }, 20:10:54 DEBUG "value": [ 20:10:54 DEBUG 20:10:54 DEBUG 20:10:54 DEBUG 20:10:54 DEBUG [...] 20:10:54 DEBUG 20:10:54 DEBUG 20:10:54 DEBUG 20:10:54 DEBUG "pod": "ds-idrepo-2" 20:10:54 DEBUG }, 20:10:54 DEBUG "value": [ 20:10:54 DEBUG 1713640254, 20:10:54 DEBUG "0" 20:10:54 DEBUG ] 20:10:54 DEBUG } 20:10:54 DEBUG ] 20:10:54 DEBUG } 20:10:54 DEBUG } 20:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640254" 20:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640254" 20:10:54 INFO [http_cmd]: http status code OK 20:10:54 DEBUG --- status code --- 20:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:54 DEBUG --- http response --- 20:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:54 DEBUG ----- output ----- 20:10:54 DEBUG { 20:10:54 DEBUG "status": "success", 20:10:54 DEBUG "data": { 20:10:54 DEBUG "resultType": "vector", 20:10:54 DEBUG "result": [ 20:10:54 DEBUG { 20:10:54 DEBUG "metric": { 20:10:54 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:54 DEBUG }, 20:10:54 DEBUG "value": [ 20:10:54 DEBUG 20:10:54 DEBUG 20:10:54 DEBUG 20:10:54 DEBUG [...] 20:10:54 DEBUG 20:10:54 DEBUG 20:10:54 DEBUG 20:10:54 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:54 DEBUG }, 20:10:54 DEBUG "value": [ 20:10:54 DEBUG 1713640254, 20:10:54 DEBUG "2838.866666666667" 20:10:54 DEBUG ] 20:10:54 DEBUG } 20:10:54 DEBUG ] 20:10:54 DEBUG } 20:10:54 DEBUG } 20:10:54 INFO [http_cmd]: http status code OK 20:10:54 DEBUG --- status code --- 20:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:54 DEBUG --- http response --- 20:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:54 DEBUG ----- output ----- 20:10:54 DEBUG { 20:10:54 DEBUG "status": "success", 20:10:54 DEBUG "data": { 20:10:54 DEBUG "resultType": "vector", 20:10:54 DEBUG "result": [ 20:10:54 DEBUG { 20:10:54 DEBUG "metric": { 20:10:54 DEBUG "pod": "ds-idrepo-0" 20:10:54 DEBUG }, 20:10:54 DEBUG "value": [ 20:10:54 DEBUG 20:10:54 DEBUG 20:10:54 DEBUG 20:10:54 DEBUG [...] 20:10:54 DEBUG 20:10:54 DEBUG 20:10:54 DEBUG 20:10:54 DEBUG "pod": "ds-idrepo-2" 20:10:54 DEBUG }, 20:10:54 DEBUG "value": [ 20:10:54 DEBUG 1713640254, 20:10:54 DEBUG "0" 20:10:54 DEBUG ] 20:10:54 DEBUG } 20:10:54 DEBUG ] 20:10:54 DEBUG } 20:10:54 DEBUG } 20:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640255" 20:10:55 INFO [http_cmd]: http status code OK 20:10:55 DEBUG --- status code --- 20:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:55 DEBUG --- http response --- 20:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:55 DEBUG ----- output ----- 20:10:55 DEBUG { 20:10:55 DEBUG "status": "success", 20:10:55 DEBUG "data": { 20:10:55 DEBUG "resultType": "vector", 20:10:55 DEBUG "result": [ 20:10:55 DEBUG { 20:10:55 DEBUG "metric": { 20:10:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:10:55 DEBUG }, 20:10:55 DEBUG "value": [ 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG [...] 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG "pod": "ds-cts-0" 20:10:55 DEBUG }, 20:10:55 DEBUG "value": [ 20:10:55 DEBUG 1713640255, 20:10:55 DEBUG "75389333.58469224" 20:10:55 DEBUG ] 20:10:55 DEBUG } 20:10:55 DEBUG ] 20:10:55 DEBUG } 20:10:55 DEBUG } 20:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640255" 20:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640255" 20:10:55 INFO [http_cmd]: http status code OK 20:10:55 DEBUG --- status code --- 20:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:55 DEBUG --- http response --- 20:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:55 DEBUG ----- output ----- 20:10:55 DEBUG { 20:10:55 DEBUG "status": "success", 20:10:55 DEBUG "data": { 20:10:55 DEBUG "resultType": "vector", 20:10:55 DEBUG "result": [ 20:10:55 DEBUG { 20:10:55 DEBUG "metric": { 20:10:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:55 DEBUG }, 20:10:55 DEBUG "value": [ 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG [...] 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:55 DEBUG }, 20:10:55 DEBUG "value": [ 20:10:55 DEBUG 1713640255, 20:10:55 DEBUG "2830.5" 20:10:55 DEBUG ] 20:10:55 DEBUG } 20:10:55 DEBUG ] 20:10:55 DEBUG } 20:10:55 DEBUG } 20:10:55 INFO [http_cmd]: http status code OK 20:10:55 DEBUG --- status code --- 20:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:55 DEBUG --- http response --- 20:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:55 DEBUG ----- output ----- 20:10:55 DEBUG { 20:10:55 DEBUG "status": "success", 20:10:55 DEBUG "data": { 20:10:55 DEBUG "resultType": "vector", 20:10:55 DEBUG "result": [ 20:10:55 DEBUG { 20:10:55 DEBUG "metric": { 20:10:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:10:55 DEBUG }, 20:10:55 DEBUG "value": [ 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG [...] 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:10:55 DEBUG }, 20:10:55 DEBUG "value": [ 20:10:55 DEBUG 1713640255, 20:10:55 DEBUG "9.560390766083625" 20:10:55 DEBUG ] 20:10:55 DEBUG } 20:10:55 DEBUG ] 20:10:55 DEBUG } 20:10:55 DEBUG } 20:10:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640255" 20:10:55 INFO [http_cmd]: http status code OK 20:10:55 DEBUG --- status code --- 20:10:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:55 DEBUG --- http response --- 20:10:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:55 DEBUG ----- output ----- 20:10:55 DEBUG { 20:10:55 DEBUG "status": "success", 20:10:55 DEBUG "data": { 20:10:55 DEBUG "resultType": "vector", 20:10:55 DEBUG "result": [ 20:10:55 DEBUG { 20:10:55 DEBUG "metric": { 20:10:55 DEBUG "instance": "10.142.0.107:9100" 20:10:55 DEBUG }, 20:10:55 DEBUG "value": [ 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG [...] 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG 20:10:55 DEBUG "instance": "10.142.0.43:9100" 20:10:55 DEBUG }, 20:10:55 DEBUG "value": [ 20:10:55 DEBUG 1713640255, 20:10:55 DEBUG "3689521435648" 20:10:55 DEBUG ] 20:10:55 DEBUG } 20:10:55 DEBUG ] 20:10:55 DEBUG } 20:10:55 DEBUG } 20:10:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640256" 20:10:56 INFO 20:10:56 INFO [loop_until]: kubectl --namespace=xlou top pods 20:10:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:10:56 INFO [http_cmd]: http status code OK 20:10:56 DEBUG --- status code --- 20:10:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:56 DEBUG --- http response --- 20:10:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:56 DEBUG ----- output ----- 20:10:56 DEBUG { 20:10:56 DEBUG "status": "success", 20:10:56 DEBUG "data": { 20:10:56 DEBUG "resultType": "vector", 20:10:56 DEBUG "result": [ 20:10:56 DEBUG { 20:10:56 DEBUG "metric": { 20:10:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:56 DEBUG }, 20:10:56 DEBUG "value": [ 20:10:56 DEBUG 20:10:56 DEBUG 20:10:56 DEBUG 20:10:56 DEBUG [...] 20:10:56 DEBUG 20:10:56 DEBUG 20:10:56 DEBUG 20:10:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:56 DEBUG }, 20:10:56 DEBUG "value": [ 20:10:56 DEBUG 1713640256, 20:10:56 DEBUG "2830.9" 20:10:56 DEBUG ] 20:10:56 DEBUG } 20:10:56 DEBUG ] 20:10:56 DEBUG } 20:10:56 DEBUG } 20:10:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640256" 20:10:56 INFO [loop_until]: OK (rc = 0) 20:10:56 DEBUG --- stdout --- 20:10:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17759m 8659Mi am-767f6b67b5-qcqz2 18260m 9068Mi am-767f6b67b5-vbltz 20119m 8673Mi ds-cts-0 20159m 10969Mi ds-cts-1 18675m 10769Mi ds-cts-2 20339m 11055Mi ds-idrepo-0 7949m 3893Mi ds-idrepo-1 7919m 3882Mi ds-idrepo-2 8029m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 16m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9312m 817Mi 20:10:56 DEBUG --- stderr --- 20:10:56 DEBUG 20:10:56 INFO [http_cmd]: http status code OK 20:10:56 DEBUG --- status code --- 20:10:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:56 DEBUG --- http response --- 20:10:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:56 DEBUG ----- output ----- 20:10:56 DEBUG { 20:10:56 DEBUG "status": "success", 20:10:56 DEBUG "data": { 20:10:56 DEBUG "resultType": "vector", 20:10:56 DEBUG "result": [ 20:10:56 DEBUG { 20:10:56 DEBUG "metric": { 20:10:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:56 DEBUG }, 20:10:56 DEBUG "value": [ 20:10:56 DEBUG 20:10:56 DEBUG 20:10:56 DEBUG 20:10:56 DEBUG [...] 20:10:56 DEBUG 20:10:56 DEBUG 20:10:56 DEBUG 20:10:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:56 DEBUG }, 20:10:56 DEBUG "value": [ 20:10:56 DEBUG 1713640256, 20:10:56 DEBUG "2829.4666666666667" 20:10:56 DEBUG ] 20:10:56 DEBUG } 20:10:56 DEBUG ] 20:10:56 DEBUG } 20:10:56 DEBUG } 20:10:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640256" 20:10:56 INFO [http_cmd]: http status code OK 20:10:56 DEBUG --- status code --- 20:10:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:56 DEBUG --- http response --- 20:10:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:10:56 DEBUG ----- output ----- 20:10:56 DEBUG { 20:10:56 DEBUG "status": "success", 20:10:56 DEBUG "data": { 20:10:56 DEBUG "resultType": "vector", 20:10:56 DEBUG "result": [ 20:10:56 DEBUG { 20:10:56 DEBUG "metric": { 20:10:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:10:56 DEBUG }, 20:10:56 DEBUG "value": [ 20:10:56 DEBUG 20:10:56 DEBUG 20:10:56 DEBUG 20:10:56 DEBUG [...] 20:10:56 DEBUG 20:10:56 DEBUG 20:10:56 DEBUG 20:10:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:10:56 DEBUG }, 20:10:56 DEBUG "value": [ 20:10:56 DEBUG 1713640256, 20:10:56 DEBUG "0" 20:10:56 DEBUG ] 20:10:56 DEBUG } 20:10:56 DEBUG ] 20:10:56 DEBUG } 20:10:56 DEBUG } 20:10:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640257" 20:10:57 INFO [http_cmd]: http status code OK 20:10:57 DEBUG --- status code --- 20:10:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:57 DEBUG --- http response --- 20:10:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:57 DEBUG ----- output ----- 20:10:57 DEBUG { 20:10:57 DEBUG "status": "success", 20:10:57 DEBUG "data": { 20:10:57 DEBUG "resultType": "vector", 20:10:57 DEBUG "result": [ 20:10:57 DEBUG { 20:10:57 DEBUG "metric": { 20:10:57 DEBUG "pod": "ds-idrepo-0" 20:10:57 DEBUG }, 20:10:57 DEBUG "value": [ 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG [...] 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG "pod": "ds-idrepo-2" 20:10:57 DEBUG }, 20:10:57 DEBUG "value": [ 20:10:57 DEBUG 1713640257, 20:10:57 DEBUG "0" 20:10:57 DEBUG ] 20:10:57 DEBUG } 20:10:57 DEBUG ] 20:10:57 DEBUG } 20:10:57 DEBUG } 20:10:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640257" 20:10:57 INFO [http_cmd]: http status code OK 20:10:57 DEBUG --- status code --- 20:10:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:57 DEBUG --- http response --- 20:10:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:10:57 DEBUG ----- output ----- 20:10:57 DEBUG { 20:10:57 DEBUG "status": "success", 20:10:57 DEBUG "data": { 20:10:57 DEBUG "resultType": "vector", 20:10:57 DEBUG "result": [ 20:10:57 DEBUG { 20:10:57 DEBUG "metric": { 20:10:57 DEBUG "pod": "ds-cts-0" 20:10:57 DEBUG }, 20:10:57 DEBUG "value": [ 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG [...] 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG "pod": "ds-cts-2" 20:10:57 DEBUG }, 20:10:57 DEBUG "value": [ 20:10:57 DEBUG 1713640257, 20:10:57 DEBUG "0" 20:10:57 DEBUG ] 20:10:57 DEBUG } 20:10:57 DEBUG ] 20:10:57 DEBUG } 20:10:57 DEBUG } 20:10:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640257" 20:10:57 INFO [http_cmd]: http status code OK 20:10:57 DEBUG --- status code --- 20:10:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:57 DEBUG --- http response --- 20:10:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:10:57 DEBUG ----- output ----- 20:10:57 DEBUG { 20:10:57 DEBUG "status": "success", 20:10:57 DEBUG "data": { 20:10:57 DEBUG "resultType": "vector", 20:10:57 DEBUG "result": [ 20:10:57 DEBUG { 20:10:57 DEBUG "metric": { 20:10:57 DEBUG "pod": "ds-idrepo-0" 20:10:57 DEBUG }, 20:10:57 DEBUG "value": [ 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG [...] 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG "pod": "ds-cts-2" 20:10:57 DEBUG }, 20:10:57 DEBUG "value": [ 20:10:57 DEBUG 1713640257, 20:10:57 DEBUG "0" 20:10:57 DEBUG ] 20:10:57 DEBUG } 20:10:57 DEBUG ] 20:10:57 DEBUG } 20:10:57 DEBUG } 20:10:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640257" 20:10:57 INFO [http_cmd]: http status code OK 20:10:57 DEBUG --- status code --- 20:10:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:57 DEBUG --- http response --- 20:10:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:57 DEBUG ----- output ----- 20:10:57 DEBUG { 20:10:57 DEBUG "status": "success", 20:10:57 DEBUG "data": { 20:10:57 DEBUG "resultType": "vector", 20:10:57 DEBUG "result": [ 20:10:57 DEBUG { 20:10:57 DEBUG "metric": { 20:10:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:57 DEBUG }, 20:10:57 DEBUG "value": [ 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG [...] 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG 20:10:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:57 DEBUG }, 20:10:57 DEBUG "value": [ 20:10:57 DEBUG 1713640257, 20:10:57 DEBUG "510194" 20:10:57 DEBUG ] 20:10:57 DEBUG } 20:10:57 DEBUG ] 20:10:57 DEBUG } 20:10:57 DEBUG } 20:10:58 INFO 20:10:58 INFO [loop_until]: kubectl --namespace=xlou top node 20:10:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:10:58 INFO [loop_until]: OK (rc = 0) 20:10:58 DEBUG --- stdout --- 20:10:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6209Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8190m 27% 7650Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18161m 60% 11379Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3342Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19502m 65% 14145Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5222Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18475m 61% 11950Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15685m 52% 2963Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20972m 70% 10901Mi 9% gke-xlou-cdm-ds-0f601751-j488 8736m 29% 7231Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18633m 62% 15818Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8665m 29% 7862Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20600m 69% 13716Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9852m 62% 6397Mi 10% 20:10:58 DEBUG --- stderr --- 20:10:58 DEBUG 20:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640258" 20:10:58 INFO [http_cmd]: http status code OK 20:10:58 DEBUG --- status code --- 20:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:58 DEBUG --- http response --- 20:10:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:58 DEBUG ----- output ----- 20:10:58 DEBUG { 20:10:58 DEBUG "status": "success", 20:10:58 DEBUG "data": { 20:10:58 DEBUG "resultType": "vector", 20:10:58 DEBUG "result": [ 20:10:58 DEBUG { 20:10:58 DEBUG "metric": { 20:10:58 DEBUG "instance": "10.142.0.107:9100" 20:10:58 DEBUG }, 20:10:58 DEBUG "value": [ 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG [...] 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG "instance": "10.142.0.43:9100" 20:10:58 DEBUG }, 20:10:58 DEBUG "value": [ 20:10:58 DEBUG 1713640258, 20:10:58 DEBUG "0.25726666666672504" 20:10:58 DEBUG ] 20:10:58 DEBUG } 20:10:58 DEBUG ] 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640258" 20:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640258" 20:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640258" 20:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640258" 20:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640258" 20:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640258" 20:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640258" 20:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640258" 20:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640258" 20:10:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640258" 20:10:58 INFO [http_cmd]: http status code OK 20:10:58 DEBUG --- status code --- 20:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:58 INFO [http_cmd]: http status code OK 20:10:58 INFO [http_cmd]: http status code OK 20:10:58 DEBUG --- status code --- 20:10:58 INFO [http_cmd]: http status code OK 20:10:58 INFO [http_cmd]: http status code OK 20:10:58 DEBUG --- http response --- 20:10:58 DEBUG --- status code --- 20:10:58 INFO [http_cmd]: http status code OK 20:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:58 DEBUG --- status code --- 20:10:58 DEBUG --- status code --- 20:10:58 INFO [http_cmd]: http status code OK 20:10:58 INFO [http_cmd]: http status code OK 20:10:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:10:58 INFO [http_cmd]: http status code OK 20:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:58 DEBUG --- status code --- 20:10:58 DEBUG --- http response --- 20:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:58 DEBUG --- status code --- 20:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:58 DEBUG ----- output ----- 20:10:58 DEBUG --- status code --- 20:10:58 DEBUG --- http response --- 20:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:58 INFO [http_cmd]: http status code OK 20:10:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:10:58 DEBUG --- http response --- 20:10:58 DEBUG --- http response --- 20:10:58 DEBUG --- http response --- 20:10:58 DEBUG --- status code --- 20:10:58 DEBUG { 20:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:58 DEBUG --- http response --- 20:10:58 DEBUG --- status code --- 20:10:58 DEBUG ----- output ----- 20:10:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:58 DEBUG "status": "success", 20:10:58 DEBUG --- http response --- 20:10:58 DEBUG ----- output ----- 20:10:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:10:58 DEBUG { 20:10:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:10:58 DEBUG ----- output ----- 20:10:58 DEBUG ----- output ----- 20:10:58 DEBUG ----- output ----- 20:10:58 DEBUG --- http response --- 20:10:58 DEBUG "data": { 20:10:58 DEBUG { 20:10:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:58 DEBUG --- http response --- 20:10:58 DEBUG "status": "success", 20:10:58 DEBUG { 20:10:58 DEBUG ----- output ----- 20:10:58 DEBUG { 20:10:58 DEBUG { 20:10:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:10:58 DEBUG "resultType": "vector", 20:10:58 DEBUG "status": "success", 20:10:58 DEBUG ----- output ----- 20:10:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:10:58 DEBUG "data": { 20:10:58 DEBUG "status": "success", 20:10:58 DEBUG { 20:10:58 DEBUG "status": "success", 20:10:58 DEBUG "status": "success", 20:10:58 DEBUG ----- output ----- 20:10:58 DEBUG "result": [ 20:10:58 DEBUG "data": { 20:10:58 DEBUG { 20:10:58 DEBUG ----- output ----- 20:10:58 DEBUG "resultType": "vector", 20:10:58 DEBUG "data": { 20:10:58 DEBUG "status": "success", 20:10:58 DEBUG "data": { 20:10:58 DEBUG "data": { 20:10:58 DEBUG { 20:10:58 DEBUG { 20:10:58 DEBUG "resultType": "vector", 20:10:58 DEBUG "status": "success", 20:10:58 DEBUG { 20:10:58 DEBUG "result": [ 20:10:58 DEBUG "resultType": "vector", 20:10:58 DEBUG "data": { 20:10:58 DEBUG "resultType": "vector", 20:10:58 DEBUG "resultType": "vector", 20:10:58 DEBUG "status": "success", 20:10:58 DEBUG "metric": { 20:10:58 DEBUG "pod": "ds-idrepo-1" 20:10:58 DEBUG "data": { 20:10:58 DEBUG "result": [ 20:10:58 DEBUG { 20:10:58 DEBUG "result": [ 20:10:58 DEBUG "resultType": "vector", 20:10:58 DEBUG "result": [ 20:10:58 DEBUG "result": [ 20:10:58 DEBUG "data": { 20:10:58 DEBUG "resultType": "vector", 20:10:58 DEBUG "status": "success", 20:10:58 DEBUG "resultType": "vector", 20:10:58 DEBUG { 20:10:58 DEBUG "metric": { 20:10:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:58 DEBUG { 20:10:58 DEBUG { 20:10:58 DEBUG "result": [ 20:10:58 DEBUG "result": [ 20:10:58 DEBUG "data": { 20:10:58 DEBUG "result": [ 20:10:58 DEBUG { 20:10:58 DEBUG "metric": { 20:10:58 DEBUG }, 20:10:58 DEBUG { 20:10:58 DEBUG "metric": { 20:10:58 DEBUG "metric": { 20:10:58 DEBUG { 20:10:58 DEBUG { 20:10:58 DEBUG "resultType": "vector", 20:10:58 DEBUG "metric": { 20:10:58 DEBUG }, 20:10:58 DEBUG "pod": "ds-cts-0" 20:10:58 DEBUG "value": [ 20:10:58 DEBUG "metric": { 20:10:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:10:58 DEBUG "pod": "ds-cts-0" 20:10:58 DEBUG }, 20:10:58 DEBUG "value": [ 20:10:58 DEBUG "result": [ 20:10:58 DEBUG "instance": "10.142.0.107:9100" 20:10:58 DEBUG }, 20:10:58 DEBUG }, 20:10:58 DEBUG 20:10:58 DEBUG "instance": "10.142.0.107:9100" 20:10:58 DEBUG }, 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG "metric": { 20:10:58 DEBUG { 20:10:58 DEBUG "value": [ 20:10:58 DEBUG "value": [ 20:10:58 DEBUG 20:10:58 DEBUG "value": [ 20:10:58 DEBUG }, 20:10:58 DEBUG "value": [ 20:10:58 DEBUG 20:10:58 DEBUG "metric": { 20:10:58 DEBUG "pod": "ds-idrepo-1" 20:10:58 DEBUG "metric": { 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG [...] 20:10:58 DEBUG "value": [ 20:10:58 DEBUG 20:10:58 DEBUG [...] 20:10:58 DEBUG 20:10:58 DEBUG }, 20:10:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG "instance": "10.142.0.107:9100" 20:10:58 DEBUG 20:10:58 DEBUG "value": [ 20:10:58 DEBUG }, 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG }, 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG "value": [ 20:10:58 DEBUG [...] 20:10:58 DEBUG [...] 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG [...] 20:10:58 DEBUG "value": [ 20:10:58 DEBUG "pod": "ds-cts-2" 20:10:58 DEBUG }, 20:10:58 DEBUG "value": [ 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG [...] 20:10:58 DEBUG [...] 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 1713640258, 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG "0.0008000000000000007" 20:10:58 DEBUG 20:10:58 DEBUG [...] 20:10:58 DEBUG "pod": "ds-cts-2" 20:10:58 DEBUG }, 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG ] 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG }, 20:10:58 DEBUG "value": [ 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:10:58 DEBUG [...] 20:10:58 DEBUG [...] 20:10:58 DEBUG } 20:10:58 DEBUG "instance": "10.142.0.43:9100" 20:10:58 DEBUG 20:10:58 DEBUG "value": [ 20:10:58 DEBUG 1713640258, 20:10:58 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:10:58 DEBUG "instance": "10.142.0.43:9100" 20:10:58 DEBUG }, 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG ] 20:10:58 DEBUG }, 20:10:58 DEBUG 20:10:58 DEBUG 1713640258, 20:10:58 DEBUG "0.157" 20:10:58 DEBUG }, 20:10:58 DEBUG }, 20:10:58 DEBUG "value": [ 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG } 20:10:58 DEBUG "value": [ 20:10:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:10:58 DEBUG "28.53333333333333" 20:10:58 DEBUG ] 20:10:58 DEBUG "value": [ 20:10:58 DEBUG "value": [ 20:10:58 DEBUG 1713640258, 20:10:58 DEBUG 20:10:58 DEBUG 20:10:58 DEBUG } 20:10:58 DEBUG 1713640258, 20:10:58 DEBUG }, 20:10:58 DEBUG ] 20:10:58 DEBUG } 20:10:58 DEBUG 1713640258, 20:10:58 DEBUG "85680546.53333333" 20:10:58 DEBUG 1713640258, 20:10:58 DEBUG "instance": "10.142.0.43:9100" 20:10:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:10:58 DEBUG "0.0829999999999965" 20:10:58 DEBUG "value": [ 20:10:58 DEBUG 1713640258, 20:10:58 DEBUG ] 20:10:58 DEBUG "0" 20:10:58 DEBUG ] 20:10:58 DEBUG "0" 20:10:58 DEBUG }, 20:10:58 DEBUG }, 20:10:58 DEBUG ] 20:10:58 DEBUG "0" 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG ] 20:10:58 DEBUG } 20:10:58 DEBUG ] 20:10:58 DEBUG "value": [ 20:10:58 DEBUG "value": [ 20:10:58 DEBUG } 20:10:58 DEBUG ] 20:10:58 DEBUG ] 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG ] 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG 1713640258, 20:10:58 DEBUG ] 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG ] 20:10:58 DEBUG 1713640258, 20:10:58 DEBUG } 20:10:58 DEBUG "85677605.53333333" 20:10:58 DEBUG } 20:10:58 DEBUG ] 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG "953843712" 20:10:58 DEBUG ] 20:10:58 DEBUG ] 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG ] 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG ] 20:10:58 DEBUG ] 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG } 20:10:58 DEBUG } 20:11:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640313" 20:11:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640313" 20:11:53 INFO [http_cmd]: http status code OK 20:11:53 DEBUG --- status code --- 20:11:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:53 DEBUG --- http response --- 20:11:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:11:53 ERROR [http_cmd]: ERROR 20:11:53 ERROR --- status code --- 20:11:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:11:53 ERROR --- http response --- 20:11:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:11:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640314" 20:11:54 INFO [http_cmd]: http status code OK 20:11:54 DEBUG --- status code --- 20:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:54 DEBUG --- http response --- 20:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:54 DEBUG ----- output ----- 20:11:54 DEBUG { 20:11:54 DEBUG "status": "success", 20:11:54 DEBUG "data": { 20:11:54 DEBUG "resultType": "vector", 20:11:54 DEBUG "result": [ 20:11:54 DEBUG { 20:11:54 DEBUG "metric": { 20:11:54 DEBUG "pod": "ds-idrepo-0" 20:11:54 DEBUG }, 20:11:54 DEBUG "value": [ 20:11:54 DEBUG 20:11:54 DEBUG 20:11:54 DEBUG 20:11:54 DEBUG [...] 20:11:54 DEBUG 20:11:54 DEBUG 20:11:54 DEBUG 20:11:54 DEBUG "pod": "ds-idrepo-2" 20:11:54 DEBUG }, 20:11:54 DEBUG "value": [ 20:11:54 DEBUG 1713640314, 20:11:54 DEBUG "0" 20:11:54 DEBUG ] 20:11:54 DEBUG } 20:11:54 DEBUG ] 20:11:54 DEBUG } 20:11:54 DEBUG } 20:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640315" 20:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640315" 20:11:55 INFO [http_cmd]: http status code OK 20:11:55 DEBUG --- status code --- 20:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:55 DEBUG --- http response --- 20:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:55 DEBUG ----- output ----- 20:11:55 DEBUG { 20:11:55 DEBUG "status": "success", 20:11:55 DEBUG "data": { 20:11:55 DEBUG "resultType": "vector", 20:11:55 DEBUG "result": [ 20:11:55 DEBUG { 20:11:55 DEBUG "metric": { 20:11:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:55 DEBUG }, 20:11:55 DEBUG "value": [ 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG [...] 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:55 DEBUG }, 20:11:55 DEBUG "value": [ 20:11:55 DEBUG 1713640315, 20:11:55 DEBUG "2831.0333333333333" 20:11:55 DEBUG ] 20:11:55 DEBUG } 20:11:55 DEBUG ] 20:11:55 DEBUG } 20:11:55 DEBUG } 20:11:55 INFO [http_cmd]: http status code OK 20:11:55 DEBUG --- status code --- 20:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:55 DEBUG --- http response --- 20:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:55 DEBUG ----- output ----- 20:11:55 DEBUG { 20:11:55 DEBUG "status": "success", 20:11:55 DEBUG "data": { 20:11:55 DEBUG "resultType": "vector", 20:11:55 DEBUG "result": [ 20:11:55 DEBUG { 20:11:55 DEBUG "metric": { 20:11:55 DEBUG "pod": "ds-idrepo-0" 20:11:55 DEBUG }, 20:11:55 DEBUG "value": [ 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG [...] 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG "pod": "ds-idrepo-2" 20:11:55 DEBUG }, 20:11:55 DEBUG "value": [ 20:11:55 DEBUG 1713640315, 20:11:55 DEBUG "0" 20:11:55 DEBUG ] 20:11:55 DEBUG } 20:11:55 DEBUG ] 20:11:55 DEBUG } 20:11:55 DEBUG } 20:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640315" 20:11:55 INFO [http_cmd]: http status code OK 20:11:55 DEBUG --- status code --- 20:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:55 DEBUG --- http response --- 20:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:55 DEBUG ----- output ----- 20:11:55 DEBUG { 20:11:55 DEBUG "status": "success", 20:11:55 DEBUG "data": { 20:11:55 DEBUG "resultType": "vector", 20:11:55 DEBUG "result": [ 20:11:55 DEBUG { 20:11:55 DEBUG "metric": { 20:11:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:11:55 DEBUG }, 20:11:55 DEBUG "value": [ 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG [...] 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:11:55 DEBUG }, 20:11:55 DEBUG "value": [ 20:11:55 DEBUG 1713640315, 20:11:55 DEBUG "0" 20:11:55 DEBUG ] 20:11:55 DEBUG } 20:11:55 DEBUG ] 20:11:55 DEBUG } 20:11:55 DEBUG } 20:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640315" 20:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640315" 20:11:55 INFO [http_cmd]: http status code OK 20:11:55 DEBUG --- status code --- 20:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:55 DEBUG --- http response --- 20:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:55 INFO [http_cmd]: http status code OK 20:11:55 DEBUG ----- output ----- 20:11:55 DEBUG --- status code --- 20:11:55 DEBUG { 20:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:55 DEBUG "status": "success", 20:11:55 DEBUG --- http response --- 20:11:55 DEBUG "data": { 20:11:55 DEBUG "resultType": "vector", 20:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:55 DEBUG "result": [ 20:11:55 DEBUG ----- output ----- 20:11:55 DEBUG { 20:11:55 DEBUG "metric": { 20:11:55 DEBUG { 20:11:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:55 DEBUG "status": "success", 20:11:55 DEBUG }, 20:11:55 DEBUG "data": { 20:11:55 DEBUG "value": [ 20:11:55 DEBUG "resultType": "vector", 20:11:55 DEBUG 20:11:55 DEBUG "result": [ 20:11:55 DEBUG 20:11:55 DEBUG { 20:11:55 DEBUG 20:11:55 DEBUG "metric": { 20:11:55 DEBUG [...] 20:11:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:11:55 DEBUG 20:11:55 DEBUG }, 20:11:55 DEBUG 20:11:55 DEBUG "value": [ 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:55 DEBUG 20:11:55 DEBUG }, 20:11:55 DEBUG 20:11:55 DEBUG "value": [ 20:11:55 DEBUG [...] 20:11:55 DEBUG 1713640315, 20:11:55 DEBUG 20:11:55 DEBUG "2828.7" 20:11:55 DEBUG 20:11:55 DEBUG ] 20:11:55 DEBUG 20:11:55 DEBUG } 20:11:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:11:55 DEBUG ] 20:11:55 DEBUG }, 20:11:55 DEBUG } 20:11:55 DEBUG "value": [ 20:11:55 DEBUG 1713640315, 20:11:55 DEBUG "9.552528128601177" 20:11:55 DEBUG ] 20:11:55 DEBUG } 20:11:55 DEBUG } 20:11:55 DEBUG ] 20:11:55 DEBUG } 20:11:55 DEBUG } 20:11:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640315" 20:11:55 INFO [http_cmd]: http status code OK 20:11:55 DEBUG --- status code --- 20:11:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:55 DEBUG --- http response --- 20:11:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:55 DEBUG ----- output ----- 20:11:55 DEBUG { 20:11:55 DEBUG "status": "success", 20:11:55 DEBUG "data": { 20:11:55 DEBUG "resultType": "vector", 20:11:55 DEBUG "result": [ 20:11:55 DEBUG { 20:11:55 DEBUG "metric": { 20:11:55 DEBUG "instance": "10.142.0.107:9100" 20:11:55 DEBUG }, 20:11:55 DEBUG "value": [ 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG [...] 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG 20:11:55 DEBUG "instance": "10.142.0.43:9100" 20:11:55 DEBUG }, 20:11:55 DEBUG "value": [ 20:11:55 DEBUG 1713640315, 20:11:55 DEBUG "3694271409152" 20:11:55 DEBUG ] 20:11:55 DEBUG } 20:11:55 DEBUG ] 20:11:55 DEBUG } 20:11:55 DEBUG } 20:11:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640316" 20:11:56 INFO [http_cmd]: http status code OK 20:11:56 DEBUG --- status code --- 20:11:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:56 DEBUG --- http response --- 20:11:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:56 DEBUG ----- output ----- 20:11:56 DEBUG { 20:11:56 DEBUG "status": "success", 20:11:56 DEBUG "data": { 20:11:56 DEBUG "resultType": "vector", 20:11:56 DEBUG "result": [ 20:11:56 DEBUG { 20:11:56 DEBUG "metric": { 20:11:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:56 DEBUG }, 20:11:56 DEBUG "value": [ 20:11:56 DEBUG 20:11:56 DEBUG 20:11:56 DEBUG 20:11:56 DEBUG [...] 20:11:56 DEBUG 20:11:56 DEBUG 20:11:56 DEBUG 20:11:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:56 DEBUG }, 20:11:56 DEBUG "value": [ 20:11:56 DEBUG 1713640316, 20:11:56 DEBUG "2828.9666666666667" 20:11:56 DEBUG ] 20:11:56 DEBUG } 20:11:56 DEBUG ] 20:11:56 DEBUG } 20:11:56 DEBUG } 20:11:56 INFO 20:11:56 INFO [loop_until]: kubectl --namespace=xlou top pods 20:11:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:11:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640316" 20:11:56 INFO [http_cmd]: http status code OK 20:11:56 DEBUG --- status code --- 20:11:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:56 DEBUG --- http response --- 20:11:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:56 DEBUG ----- output ----- 20:11:56 DEBUG { 20:11:56 DEBUG "status": "success", 20:11:56 DEBUG "data": { 20:11:56 DEBUG "resultType": "vector", 20:11:56 DEBUG "result": [ 20:11:56 DEBUG { 20:11:56 DEBUG "metric": { 20:11:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:56 DEBUG }, 20:11:56 DEBUG "value": [ 20:11:56 DEBUG 20:11:56 DEBUG 20:11:56 DEBUG 20:11:56 DEBUG [...] 20:11:56 DEBUG 20:11:56 DEBUG 20:11:56 DEBUG 20:11:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:56 DEBUG }, 20:11:56 DEBUG "value": [ 20:11:56 DEBUG 1713640316, 20:11:56 DEBUG "2829.5666666666666" 20:11:56 DEBUG ] 20:11:56 DEBUG } 20:11:56 DEBUG ] 20:11:56 DEBUG } 20:11:56 DEBUG } 20:11:56 INFO [loop_until]: OK (rc = 0) 20:11:56 DEBUG --- stdout --- 20:11:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18814m 8656Mi am-767f6b67b5-qcqz2 18314m 9067Mi am-767f6b67b5-vbltz 20195m 8668Mi ds-cts-0 19424m 10967Mi ds-cts-1 18848m 10776Mi ds-cts-2 20631m 11053Mi ds-idrepo-0 8163m 3893Mi ds-idrepo-1 8132m 3882Mi ds-idrepo-2 7921m 3828Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 14m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9474m 820Mi 20:11:56 DEBUG --- stderr --- 20:11:56 DEBUG 20:11:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640316" 20:11:56 INFO [http_cmd]: http status code OK 20:11:56 DEBUG --- status code --- 20:11:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:56 DEBUG --- http response --- 20:11:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:11:56 DEBUG ----- output ----- 20:11:56 DEBUG { 20:11:56 DEBUG "status": "success", 20:11:56 DEBUG "data": { 20:11:56 DEBUG "resultType": "vector", 20:11:56 DEBUG "result": [ 20:11:56 DEBUG { 20:11:56 DEBUG "metric": { 20:11:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:11:56 DEBUG }, 20:11:56 DEBUG "value": [ 20:11:56 DEBUG 20:11:56 DEBUG 20:11:56 DEBUG 20:11:56 DEBUG [...] 20:11:56 DEBUG 20:11:56 DEBUG 20:11:56 DEBUG 20:11:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:11:56 DEBUG }, 20:11:56 DEBUG "value": [ 20:11:56 DEBUG 1713640316, 20:11:56 DEBUG "0.00002703578838172119" 20:11:56 DEBUG ] 20:11:56 DEBUG } 20:11:56 DEBUG ] 20:11:56 DEBUG } 20:11:56 DEBUG } 20:11:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640317" 20:11:57 INFO [http_cmd]: http status code OK 20:11:57 DEBUG --- status code --- 20:11:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:57 DEBUG --- http response --- 20:11:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:57 DEBUG ----- output ----- 20:11:57 DEBUG { 20:11:57 DEBUG "status": "success", 20:11:57 DEBUG "data": { 20:11:57 DEBUG "resultType": "vector", 20:11:57 DEBUG "result": [ 20:11:57 DEBUG { 20:11:57 DEBUG "metric": { 20:11:57 DEBUG "pod": "ds-idrepo-0" 20:11:57 DEBUG }, 20:11:57 DEBUG "value": [ 20:11:57 DEBUG 20:11:57 DEBUG 20:11:57 DEBUG 20:11:57 DEBUG [...] 20:11:57 DEBUG 20:11:57 DEBUG 20:11:57 DEBUG 20:11:57 DEBUG "pod": "ds-idrepo-2" 20:11:57 DEBUG }, 20:11:57 DEBUG "value": [ 20:11:57 DEBUG 1713640317, 20:11:57 DEBUG "0" 20:11:57 DEBUG ] 20:11:57 DEBUG } 20:11:57 DEBUG ] 20:11:57 DEBUG } 20:11:57 DEBUG } 20:11:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640317" 20:11:57 INFO [http_cmd]: http status code OK 20:11:57 DEBUG --- status code --- 20:11:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:57 DEBUG --- http response --- 20:11:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:11:57 DEBUG ----- output ----- 20:11:57 DEBUG { 20:11:57 DEBUG "status": "success", 20:11:57 DEBUG "data": { 20:11:57 DEBUG "resultType": "vector", 20:11:57 DEBUG "result": [ 20:11:57 DEBUG { 20:11:57 DEBUG "metric": { 20:11:57 DEBUG "pod": "ds-cts-0" 20:11:57 DEBUG }, 20:11:57 DEBUG "value": [ 20:11:57 DEBUG 20:11:57 DEBUG 20:11:57 DEBUG 20:11:57 DEBUG [...] 20:11:57 DEBUG 20:11:57 DEBUG 20:11:57 DEBUG 20:11:57 DEBUG "pod": "ds-cts-2" 20:11:57 DEBUG }, 20:11:57 DEBUG "value": [ 20:11:57 DEBUG 1713640317, 20:11:57 DEBUG "0" 20:11:57 DEBUG ] 20:11:57 DEBUG } 20:11:57 DEBUG ] 20:11:57 DEBUG } 20:11:57 DEBUG } 20:11:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640317" 20:11:57 INFO [http_cmd]: http status code OK 20:11:57 DEBUG --- status code --- 20:11:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:57 DEBUG --- http response --- 20:11:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:11:57 DEBUG ----- output ----- 20:11:57 DEBUG { 20:11:57 DEBUG "status": "success", 20:11:57 DEBUG "data": { 20:11:57 DEBUG "resultType": "vector", 20:11:57 DEBUG "result": [ 20:11:57 DEBUG { 20:11:57 DEBUG "metric": { 20:11:57 DEBUG "pod": "ds-idrepo-0" 20:11:57 DEBUG }, 20:11:57 DEBUG "value": [ 20:11:57 DEBUG 20:11:57 DEBUG 20:11:57 DEBUG 20:11:57 DEBUG [...] 20:11:57 DEBUG 20:11:57 DEBUG 20:11:57 DEBUG 20:11:57 DEBUG "pod": "ds-cts-2" 20:11:57 DEBUG }, 20:11:57 DEBUG "value": [ 20:11:57 DEBUG 1713640317, 20:11:57 DEBUG "0" 20:11:57 DEBUG ] 20:11:57 DEBUG } 20:11:57 DEBUG ] 20:11:57 DEBUG } 20:11:57 DEBUG } 20:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640318" 20:11:58 INFO [http_cmd]: http status code OK 20:11:58 DEBUG --- status code --- 20:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:58 DEBUG --- http response --- 20:11:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:58 DEBUG ----- output ----- 20:11:58 DEBUG { 20:11:58 DEBUG "status": "success", 20:11:58 DEBUG "data": { 20:11:58 DEBUG "resultType": "vector", 20:11:58 DEBUG "result": [ 20:11:58 DEBUG { 20:11:58 DEBUG "metric": { 20:11:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:58 DEBUG }, 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG [...] 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:58 DEBUG }, 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 1713640318, 20:11:58 DEBUG "510492" 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG } 20:11:58 INFO 20:11:58 INFO [loop_until]: kubectl --namespace=xlou top node 20:11:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:11:58 INFO [loop_until]: OK (rc = 0) 20:11:58 DEBUG --- stdout --- 20:11:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6217Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8186m 27% 7651Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18223m 61% 11378Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3336Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19911m 66% 14158Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5187Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18501m 61% 11951Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15677m 52% 2962Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20202m 67% 10892Mi 9% gke-xlou-cdm-ds-0f601751-j488 8724m 29% 7221Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19016m 63% 15815Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8609m 28% 7851Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20265m 67% 13720Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9787m 61% 6397Mi 10% 20:11:58 DEBUG --- stderr --- 20:11:58 DEBUG 20:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640318" 20:11:58 INFO [http_cmd]: http status code OK 20:11:58 DEBUG --- status code --- 20:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:58 DEBUG --- http response --- 20:11:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:58 DEBUG ----- output ----- 20:11:58 DEBUG { 20:11:58 DEBUG "status": "success", 20:11:58 DEBUG "data": { 20:11:58 DEBUG "resultType": "vector", 20:11:58 DEBUG "result": [ 20:11:58 DEBUG { 20:11:58 DEBUG "metric": { 20:11:58 DEBUG "instance": "10.142.0.107:9100" 20:11:58 DEBUG }, 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG [...] 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG "instance": "10.142.0.43:9100" 20:11:58 DEBUG }, 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 1713640318, 20:11:58 DEBUG "0.25896666666662327" 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG } 20:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640318" 20:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640318" 20:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640318" 20:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640318" 20:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640318" 20:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640318" 20:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640318" 20:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640318" 20:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640318" 20:11:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640318" 20:11:58 INFO [http_cmd]: http status code OK 20:11:58 DEBUG --- status code --- 20:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:58 DEBUG --- http response --- 20:11:58 INFO [http_cmd]: http status code OK 20:11:58 DEBUG --- status code --- 20:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:58 DEBUG --- http response --- 20:11:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:58 INFO [http_cmd]: http status code OK 20:11:58 INFO [http_cmd]: http status code OK 20:11:58 INFO [http_cmd]: http status code OK 20:11:58 DEBUG ----- output ----- 20:11:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:11:58 INFO [http_cmd]: http status code OK 20:11:58 DEBUG --- status code --- 20:11:58 INFO [http_cmd]: http status code OK 20:11:58 INFO [http_cmd]: http status code OK 20:11:58 DEBUG --- status code --- 20:11:58 DEBUG --- status code --- 20:11:58 DEBUG { 20:11:58 DEBUG ----- output ----- 20:11:58 INFO [http_cmd]: http status code OK 20:11:58 INFO [http_cmd]: http status code OK 20:11:58 DEBUG --- status code --- 20:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:58 DEBUG --- status code --- 20:11:58 DEBUG --- status code --- 20:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:58 DEBUG "status": "success", 20:11:58 DEBUG { 20:11:58 DEBUG --- status code --- 20:11:58 DEBUG --- status code --- 20:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:58 DEBUG --- http response --- 20:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:58 DEBUG --- http response --- 20:11:58 DEBUG --- http response --- 20:11:58 DEBUG "data": { 20:11:58 DEBUG "status": "success", 20:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:11:58 DEBUG --- http response --- 20:11:58 DEBUG --- http response --- 20:11:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:58 DEBUG --- http response --- 20:11:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:11:58 DEBUG "resultType": "vector", 20:11:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:11:58 DEBUG "data": { 20:11:58 DEBUG --- http response --- 20:11:58 DEBUG --- http response --- 20:11:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:58 DEBUG ----- output ----- 20:11:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:11:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:58 DEBUG ----- output ----- 20:11:58 DEBUG "result": [ 20:11:58 DEBUG ----- output ----- 20:11:58 DEBUG "resultType": "vector", 20:11:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:11:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:11:58 DEBUG ----- output ----- 20:11:58 DEBUG { 20:11:58 DEBUG ----- output ----- 20:11:58 DEBUG ----- output ----- 20:11:58 DEBUG { 20:11:58 DEBUG { 20:11:58 DEBUG { 20:11:58 DEBUG "result": [ 20:11:58 DEBUG ----- output ----- 20:11:58 DEBUG ----- output ----- 20:11:58 DEBUG { 20:11:58 DEBUG "status": "success", 20:11:58 DEBUG { 20:11:58 DEBUG { 20:11:58 DEBUG "status": "success", 20:11:58 DEBUG "metric": { 20:11:58 DEBUG "status": "success", 20:11:58 DEBUG "data": { 20:11:58 DEBUG "resultType": "vector", 20:11:58 DEBUG { 20:11:58 DEBUG "status": "success", 20:11:58 DEBUG "data": { 20:11:58 DEBUG "status": "success", 20:11:58 DEBUG "data": { 20:11:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:58 DEBUG "status": "success", 20:11:58 DEBUG { 20:11:58 DEBUG "result": [ 20:11:58 DEBUG "metric": { 20:11:58 DEBUG { 20:11:58 DEBUG "data": { 20:11:58 DEBUG "resultType": "vector", 20:11:58 DEBUG "data": { 20:11:58 DEBUG "resultType": "vector", 20:11:58 DEBUG }, 20:11:58 DEBUG "data": { 20:11:58 DEBUG "status": "success", 20:11:58 DEBUG { 20:11:58 DEBUG "pod": "ds-cts-0" 20:11:58 DEBUG "status": "success", 20:11:58 DEBUG "resultType": "vector", 20:11:58 DEBUG "result": [ 20:11:58 DEBUG "resultType": "vector", 20:11:58 DEBUG "result": [ 20:11:58 DEBUG "value": [ 20:11:58 DEBUG "resultType": "vector", 20:11:58 DEBUG "data": { 20:11:58 DEBUG "metric": { 20:11:58 DEBUG }, 20:11:58 DEBUG "data": { 20:11:58 DEBUG "result": [ 20:11:58 DEBUG { 20:11:58 DEBUG "result": [ 20:11:58 DEBUG { 20:11:58 DEBUG 20:11:58 DEBUG "result": [ 20:11:58 DEBUG "resultType": "vector", 20:11:58 DEBUG "pod": "ds-cts-0" 20:11:58 DEBUG "value": [ 20:11:58 DEBUG "resultType": "vector", 20:11:58 DEBUG { 20:11:58 DEBUG "metric": { 20:11:58 DEBUG { 20:11:58 DEBUG "metric": { 20:11:58 DEBUG 20:11:58 DEBUG { 20:11:58 DEBUG "result": [ 20:11:58 DEBUG }, 20:11:58 DEBUG 20:11:58 DEBUG "result": [ 20:11:58 DEBUG "metric": { 20:11:58 DEBUG "instance": "10.142.0.107:9100" 20:11:58 DEBUG }, 20:11:58 DEBUG "instance": "10.142.0.107:9100" 20:11:58 DEBUG 20:11:58 DEBUG "metric": { 20:11:58 DEBUG { 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 20:11:58 DEBUG { 20:11:58 DEBUG "instance": "10.142.0.107:9100" 20:11:58 DEBUG "metric": { 20:11:58 DEBUG "value": [ 20:11:58 DEBUG }, 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG [...] 20:11:58 DEBUG 20:11:58 DEBUG "metric": { 20:11:58 DEBUG }, 20:11:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG "metric": { 20:11:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:11:58 DEBUG "value": [ 20:11:58 DEBUG }, 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG [...] 20:11:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:11:58 DEBUG }, 20:11:58 DEBUG 20:11:58 DEBUG }, 20:11:58 DEBUG 20:11:58 DEBUG [...] 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:58 DEBUG 20:11:58 DEBUG }, 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 20:11:58 DEBUG "value": [ 20:11:58 DEBUG [...] 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG [...] 20:11:58 DEBUG }, 20:11:58 DEBUG 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG [...] 20:11:58 DEBUG 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 1713640318, 20:11:58 DEBUG "0.157" 20:11:58 DEBUG 20:11:58 DEBUG [...] 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG ] 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG "instance": "10.142.0.43:9100" 20:11:58 DEBUG 20:11:58 DEBUG "pod": "ds-cts-2" 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG } 20:11:58 DEBUG [...] 20:11:58 DEBUG 20:11:58 DEBUG [...] 20:11:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:11:58 DEBUG }, 20:11:58 DEBUG "instance": "10.142.0.43:9100" 20:11:58 DEBUG }, 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG ] 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG }, 20:11:58 DEBUG "value": [ 20:11:58 DEBUG }, 20:11:58 DEBUG "value": [ 20:11:58 DEBUG [...] 20:11:58 DEBUG "pod": "ds-cts-2" 20:11:58 DEBUG } 20:11:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:11:58 DEBUG 20:11:58 DEBUG 20:11:58 DEBUG "instance": "10.142.0.43:9100" 20:11:58 DEBUG 1713640318, 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 1713640318, 20:11:58 DEBUG 20:11:58 DEBUG }, 20:11:58 DEBUG } 20:11:58 DEBUG }, 20:11:58 DEBUG "value": [ 20:11:58 DEBUG }, 20:11:58 DEBUG "value": [ 20:11:58 DEBUG "0.0886666666666656" 20:11:58 DEBUG 1713640318, 20:11:58 DEBUG "0.0005865333333333331" 20:11:58 DEBUG 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 1713640318, 20:11:58 DEBUG 20:11:58 DEBUG "value": [ 20:11:58 DEBUG 1713640318, 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG ] 20:11:58 DEBUG 20:11:58 DEBUG 1713640318, 20:11:58 DEBUG "30.433333333333334" 20:11:58 DEBUG ] 20:11:58 DEBUG 1713640318, 20:11:58 DEBUG "0" 20:11:58 DEBUG "85958657.1" 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:11:58 DEBUG } 20:11:58 DEBUG 20:11:58 DEBUG "0" 20:11:58 DEBUG "85956173.03333333" 20:11:58 DEBUG ] 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG } 20:11:58 DEBUG }, 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG ] 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG } 20:11:58 DEBUG ] 20:11:58 DEBUG "value": [ 20:11:58 DEBUG } 20:11:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:11:58 DEBUG } 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG } 20:11:58 DEBUG ] 20:11:58 DEBUG 1713640318, 20:11:58 DEBUG }, 20:11:58 DEBUG } 20:11:58 DEBUG } 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG } 20:11:58 DEBUG "957497344" 20:11:58 DEBUG "value": [ 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG } 20:11:58 DEBUG ] 20:11:58 DEBUG 1713640318, 20:11:58 DEBUG "0" 20:11:58 DEBUG } 20:11:58 DEBUG } 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG } 20:11:58 DEBUG } 20:11:58 DEBUG ] 20:11:58 DEBUG } 20:11:58 DEBUG } 20:11:58 DEBUG } 20:12:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640373" 20:12:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640373" 20:12:53 INFO [http_cmd]: http status code OK 20:12:53 DEBUG --- status code --- 20:12:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:53 DEBUG --- http response --- 20:12:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:12:53 ERROR [http_cmd]: ERROR 20:12:53 ERROR --- status code --- 20:12:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:12:53 ERROR --- http response --- 20:12:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:12:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640375" 20:12:55 INFO [http_cmd]: http status code OK 20:12:55 DEBUG --- status code --- 20:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:55 DEBUG --- http response --- 20:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:55 DEBUG ----- output ----- 20:12:55 DEBUG { 20:12:55 DEBUG "status": "success", 20:12:55 DEBUG "data": { 20:12:55 DEBUG "resultType": "vector", 20:12:55 DEBUG "result": [ 20:12:55 DEBUG { 20:12:55 DEBUG "metric": { 20:12:55 DEBUG "pod": "ds-idrepo-0" 20:12:55 DEBUG }, 20:12:55 DEBUG "value": [ 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG [...] 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG "pod": "ds-idrepo-2" 20:12:55 DEBUG }, 20:12:55 DEBUG "value": [ 20:12:55 DEBUG 1713640375, 20:12:55 DEBUG "0" 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG } 20:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640375" 20:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640375" 20:12:55 INFO [http_cmd]: http status code OK 20:12:55 DEBUG --- status code --- 20:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:55 DEBUG --- http response --- 20:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:55 DEBUG ----- output ----- 20:12:55 DEBUG { 20:12:55 DEBUG "status": "success", 20:12:55 DEBUG "data": { 20:12:55 DEBUG "resultType": "vector", 20:12:55 DEBUG "result": [ 20:12:55 DEBUG { 20:12:55 DEBUG "metric": { 20:12:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:55 DEBUG }, 20:12:55 DEBUG "value": [ 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG [...] 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:55 DEBUG }, 20:12:55 DEBUG "value": [ 20:12:55 DEBUG 1713640375, 20:12:55 DEBUG "2845.4666666666667" 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG } 20:12:55 INFO [http_cmd]: http status code OK 20:12:55 DEBUG --- status code --- 20:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:55 DEBUG --- http response --- 20:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:55 DEBUG ----- output ----- 20:12:55 DEBUG { 20:12:55 DEBUG "status": "success", 20:12:55 DEBUG "data": { 20:12:55 DEBUG "resultType": "vector", 20:12:55 DEBUG "result": [ 20:12:55 DEBUG { 20:12:55 DEBUG "metric": { 20:12:55 DEBUG "pod": "ds-idrepo-0" 20:12:55 DEBUG }, 20:12:55 DEBUG "value": [ 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG [...] 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG "pod": "ds-idrepo-2" 20:12:55 DEBUG }, 20:12:55 DEBUG "value": [ 20:12:55 DEBUG 1713640375, 20:12:55 DEBUG "0" 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG } 20:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640375" 20:12:55 INFO [http_cmd]: http status code OK 20:12:55 DEBUG --- status code --- 20:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:55 DEBUG --- http response --- 20:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:12:55 DEBUG ----- output ----- 20:12:55 DEBUG { 20:12:55 DEBUG "status": "success", 20:12:55 DEBUG "data": { 20:12:55 DEBUG "resultType": "vector", 20:12:55 DEBUG "result": [ 20:12:55 DEBUG { 20:12:55 DEBUG "metric": { 20:12:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:12:55 DEBUG }, 20:12:55 DEBUG "value": [ 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG [...] 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:12:55 DEBUG }, 20:12:55 DEBUG "value": [ 20:12:55 DEBUG 1713640375, 20:12:55 DEBUG "0" 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG } 20:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640375" 20:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640375" 20:12:55 INFO [http_cmd]: http status code OK 20:12:55 DEBUG --- status code --- 20:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:55 DEBUG --- http response --- 20:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:55 INFO [http_cmd]: http status code OK 20:12:55 DEBUG ----- output ----- 20:12:55 DEBUG --- status code --- 20:12:55 DEBUG { 20:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:55 DEBUG "status": "success", 20:12:55 DEBUG --- http response --- 20:12:55 DEBUG "data": { 20:12:55 DEBUG "resultType": "vector", 20:12:55 DEBUG "result": [ 20:12:55 DEBUG { 20:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:55 DEBUG "metric": { 20:12:55 DEBUG ----- output ----- 20:12:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:12:55 DEBUG { 20:12:55 DEBUG }, 20:12:55 DEBUG "status": "success", 20:12:55 DEBUG "value": [ 20:12:55 DEBUG "data": { 20:12:55 DEBUG 20:12:55 DEBUG "resultType": "vector", 20:12:55 DEBUG 20:12:55 DEBUG "result": [ 20:12:55 DEBUG 20:12:55 DEBUG { 20:12:55 DEBUG [...] 20:12:55 DEBUG "metric": { 20:12:55 DEBUG 20:12:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:55 DEBUG 20:12:55 DEBUG }, 20:12:55 DEBUG 20:12:55 DEBUG "value": [ 20:12:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:12:55 DEBUG 20:12:55 DEBUG }, 20:12:55 DEBUG 20:12:55 DEBUG "value": [ 20:12:55 DEBUG 20:12:55 DEBUG 1713640375, 20:12:55 DEBUG [...] 20:12:55 DEBUG "9.55107217683155" 20:12:55 DEBUG 20:12:55 DEBUG ] 20:12:55 DEBUG 20:12:55 DEBUG } 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG } 20:12:55 DEBUG 20:12:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:55 DEBUG }, 20:12:55 DEBUG "value": [ 20:12:55 DEBUG 1713640375, 20:12:55 DEBUG "2850.1666666666665" 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG } 20:12:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640375" 20:12:55 INFO [http_cmd]: http status code OK 20:12:55 DEBUG --- status code --- 20:12:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:55 DEBUG --- http response --- 20:12:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:55 DEBUG ----- output ----- 20:12:55 DEBUG { 20:12:55 DEBUG "status": "success", 20:12:55 DEBUG "data": { 20:12:55 DEBUG "resultType": "vector", 20:12:55 DEBUG "result": [ 20:12:55 DEBUG { 20:12:55 DEBUG "metric": { 20:12:55 DEBUG "instance": "10.142.0.107:9100" 20:12:55 DEBUG }, 20:12:55 DEBUG "value": [ 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG [...] 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG 20:12:55 DEBUG "instance": "10.142.0.43:9100" 20:12:55 DEBUG }, 20:12:55 DEBUG "value": [ 20:12:55 DEBUG 1713640375, 20:12:55 DEBUG "3698968067072" 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG ] 20:12:55 DEBUG } 20:12:55 DEBUG } 20:12:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640376" 20:12:56 INFO [http_cmd]: http status code OK 20:12:56 DEBUG --- status code --- 20:12:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:56 DEBUG --- http response --- 20:12:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:56 DEBUG ----- output ----- 20:12:56 DEBUG { 20:12:56 DEBUG "status": "success", 20:12:56 DEBUG "data": { 20:12:56 DEBUG "resultType": "vector", 20:12:56 DEBUG "result": [ 20:12:56 DEBUG { 20:12:56 DEBUG "metric": { 20:12:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:56 DEBUG }, 20:12:56 DEBUG "value": [ 20:12:56 DEBUG 20:12:56 DEBUG 20:12:56 DEBUG 20:12:56 DEBUG [...] 20:12:56 DEBUG 20:12:56 DEBUG 20:12:56 DEBUG 20:12:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:56 DEBUG }, 20:12:56 DEBUG "value": [ 20:12:56 DEBUG 1713640376, 20:12:56 DEBUG "2851.1666666666665" 20:12:56 DEBUG ] 20:12:56 DEBUG } 20:12:56 DEBUG ] 20:12:56 DEBUG } 20:12:56 DEBUG } 20:12:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640376" 20:12:56 INFO [http_cmd]: http status code OK 20:12:56 DEBUG --- status code --- 20:12:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:56 DEBUG --- http response --- 20:12:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:56 DEBUG ----- output ----- 20:12:56 DEBUG { 20:12:56 DEBUG "status": "success", 20:12:56 DEBUG "data": { 20:12:56 DEBUG "resultType": "vector", 20:12:56 DEBUG "result": [ 20:12:56 DEBUG { 20:12:56 DEBUG "metric": { 20:12:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:56 DEBUG }, 20:12:56 DEBUG "value": [ 20:12:56 DEBUG 20:12:56 DEBUG 20:12:56 DEBUG 20:12:56 DEBUG [...] 20:12:56 DEBUG 20:12:56 DEBUG 20:12:56 DEBUG 20:12:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:56 DEBUG }, 20:12:56 DEBUG "value": [ 20:12:56 DEBUG 1713640376, 20:12:56 DEBUG "2852.7" 20:12:56 DEBUG ] 20:12:56 DEBUG } 20:12:56 DEBUG ] 20:12:56 DEBUG } 20:12:56 DEBUG } 20:12:56 INFO 20:12:56 INFO [loop_until]: kubectl --namespace=xlou top pods 20:12:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:12:56 INFO [loop_until]: OK (rc = 0) 20:12:56 DEBUG --- stdout --- 20:12:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18703m 8658Mi am-767f6b67b5-qcqz2 18510m 9069Mi am-767f6b67b5-vbltz 20114m 8674Mi ds-cts-0 19825m 10967Mi ds-cts-1 18995m 10758Mi ds-cts-2 20577m 11048Mi ds-idrepo-0 7993m 3894Mi ds-idrepo-1 7906m 3882Mi ds-idrepo-2 7719m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 19m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9546m 822Mi 20:12:56 DEBUG --- stderr --- 20:12:56 DEBUG 20:12:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640376" 20:12:56 INFO [http_cmd]: http status code OK 20:12:56 DEBUG --- status code --- 20:12:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:56 DEBUG --- http response --- 20:12:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:12:56 DEBUG ----- output ----- 20:12:56 DEBUG { 20:12:56 DEBUG "status": "success", 20:12:56 DEBUG "data": { 20:12:56 DEBUG "resultType": "vector", 20:12:56 DEBUG "result": [ 20:12:56 DEBUG { 20:12:56 DEBUG "metric": { 20:12:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:12:56 DEBUG }, 20:12:56 DEBUG "value": [ 20:12:56 DEBUG 20:12:56 DEBUG 20:12:56 DEBUG 20:12:56 DEBUG [...] 20:12:56 DEBUG 20:12:56 DEBUG 20:12:56 DEBUG 20:12:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:12:56 DEBUG }, 20:12:56 DEBUG "value": [ 20:12:56 DEBUG 1713640376, 20:12:56 DEBUG "9.503722085564876" 20:12:56 DEBUG ] 20:12:56 DEBUG } 20:12:56 DEBUG ] 20:12:56 DEBUG } 20:12:56 DEBUG } 20:12:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640377" 20:12:57 INFO [http_cmd]: http status code OK 20:12:57 DEBUG --- status code --- 20:12:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:57 DEBUG --- http response --- 20:12:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:57 DEBUG ----- output ----- 20:12:57 DEBUG { 20:12:57 DEBUG "status": "success", 20:12:57 DEBUG "data": { 20:12:57 DEBUG "resultType": "vector", 20:12:57 DEBUG "result": [ 20:12:57 DEBUG { 20:12:57 DEBUG "metric": { 20:12:57 DEBUG "pod": "ds-idrepo-0" 20:12:57 DEBUG }, 20:12:57 DEBUG "value": [ 20:12:57 DEBUG 20:12:57 DEBUG 20:12:57 DEBUG 20:12:57 DEBUG [...] 20:12:57 DEBUG 20:12:57 DEBUG 20:12:57 DEBUG 20:12:57 DEBUG "pod": "ds-idrepo-2" 20:12:57 DEBUG }, 20:12:57 DEBUG "value": [ 20:12:57 DEBUG 1713640377, 20:12:57 DEBUG "0" 20:12:57 DEBUG ] 20:12:57 DEBUG } 20:12:57 DEBUG ] 20:12:57 DEBUG } 20:12:57 DEBUG } 20:12:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640377" 20:12:57 INFO [http_cmd]: http status code OK 20:12:57 DEBUG --- status code --- 20:12:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:57 DEBUG --- http response --- 20:12:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:12:57 DEBUG ----- output ----- 20:12:57 DEBUG { 20:12:57 DEBUG "status": "success", 20:12:57 DEBUG "data": { 20:12:57 DEBUG "resultType": "vector", 20:12:57 DEBUG "result": [ 20:12:57 DEBUG { 20:12:57 DEBUG "metric": { 20:12:57 DEBUG "pod": "ds-cts-0" 20:12:57 DEBUG }, 20:12:57 DEBUG "value": [ 20:12:57 DEBUG 20:12:57 DEBUG 20:12:57 DEBUG 20:12:57 DEBUG [...] 20:12:57 DEBUG 20:12:57 DEBUG 20:12:57 DEBUG 20:12:57 DEBUG "pod": "ds-cts-2" 20:12:57 DEBUG }, 20:12:57 DEBUG "value": [ 20:12:57 DEBUG 1713640377, 20:12:57 DEBUG "0" 20:12:57 DEBUG ] 20:12:57 DEBUG } 20:12:57 DEBUG ] 20:12:57 DEBUG } 20:12:57 DEBUG } 20:12:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640377" 20:12:58 INFO [http_cmd]: http status code OK 20:12:58 DEBUG --- status code --- 20:12:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:58 DEBUG --- http response --- 20:12:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:12:58 DEBUG ----- output ----- 20:12:58 DEBUG { 20:12:58 DEBUG "status": "success", 20:12:58 DEBUG "data": { 20:12:58 DEBUG "resultType": "vector", 20:12:58 DEBUG "result": [ 20:12:58 DEBUG { 20:12:58 DEBUG "metric": { 20:12:58 DEBUG "pod": "ds-idrepo-0" 20:12:58 DEBUG }, 20:12:58 DEBUG "value": [ 20:12:58 DEBUG 20:12:58 DEBUG 20:12:58 DEBUG 20:12:58 DEBUG [...] 20:12:58 DEBUG 20:12:58 DEBUG 20:12:58 DEBUG 20:12:58 DEBUG "pod": "ds-cts-2" 20:12:58 DEBUG }, 20:12:58 DEBUG "value": [ 20:12:58 DEBUG 1713640377, 20:12:58 DEBUG "0" 20:12:58 DEBUG ] 20:12:58 DEBUG } 20:12:58 DEBUG ] 20:12:58 DEBUG } 20:12:58 DEBUG } 20:12:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640378" 20:12:58 INFO [http_cmd]: http status code OK 20:12:58 DEBUG --- status code --- 20:12:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:58 DEBUG --- http response --- 20:12:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:58 DEBUG ----- output ----- 20:12:58 DEBUG { 20:12:58 DEBUG "status": "success", 20:12:58 DEBUG "data": { 20:12:58 DEBUG "resultType": "vector", 20:12:58 DEBUG "result": [ 20:12:58 DEBUG { 20:12:58 DEBUG "metric": { 20:12:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:58 DEBUG }, 20:12:58 DEBUG "value": [ 20:12:58 DEBUG 20:12:58 DEBUG 20:12:58 DEBUG 20:12:58 DEBUG [...] 20:12:58 DEBUG 20:12:58 DEBUG 20:12:58 DEBUG 20:12:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:58 DEBUG }, 20:12:58 DEBUG "value": [ 20:12:58 DEBUG 1713640378, 20:12:58 DEBUG "509778" 20:12:58 DEBUG ] 20:12:58 DEBUG } 20:12:58 DEBUG ] 20:12:58 DEBUG } 20:12:58 DEBUG } 20:12:58 INFO 20:12:58 INFO [loop_until]: kubectl --namespace=xlou top node 20:12:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:12:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640378" 20:12:58 INFO [http_cmd]: http status code OK 20:12:58 DEBUG --- status code --- 20:12:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:58 DEBUG --- http response --- 20:12:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:58 DEBUG ----- output ----- 20:12:58 DEBUG { 20:12:58 DEBUG "status": "success", 20:12:58 DEBUG "data": { 20:12:58 DEBUG "resultType": "vector", 20:12:58 DEBUG "result": [ 20:12:58 DEBUG { 20:12:58 DEBUG "metric": { 20:12:58 DEBUG "instance": "10.142.0.107:9100" 20:12:58 DEBUG }, 20:12:58 DEBUG "value": [ 20:12:58 DEBUG 20:12:58 DEBUG 20:12:58 DEBUG 20:12:58 DEBUG [...] 20:12:58 DEBUG 20:12:58 DEBUG 20:12:58 DEBUG 20:12:58 DEBUG "instance": "10.142.0.43:9100" 20:12:58 DEBUG }, 20:12:58 DEBUG "value": [ 20:12:58 DEBUG 1713640378, 20:12:58 DEBUG "0.2351333333333893" 20:12:58 DEBUG ] 20:12:58 DEBUG } 20:12:58 DEBUG ] 20:12:58 DEBUG } 20:12:58 DEBUG } 20:12:58 INFO [loop_until]: OK (rc = 0) 20:12:58 DEBUG --- stdout --- 20:12:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6202Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8258m 27% 7644Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18330m 61% 11364Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3338Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20025m 67% 14155Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5209Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18670m 62% 11953Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15826m 53% 2967Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20900m 70% 10886Mi 9% gke-xlou-cdm-ds-0f601751-j488 8831m 29% 7226Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18562m 62% 15816Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8680m 29% 7857Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19927m 66% 13682Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9829m 61% 6398Mi 10% 20:12:58 DEBUG --- stderr --- 20:12:58 DEBUG 20:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640379" 20:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640379" 20:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640379" 20:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640379" 20:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640379" 20:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640379" 20:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640379" 20:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640379" 20:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640379" 20:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640379" 20:12:59 INFO [http_cmd]: http status code OK 20:12:59 DEBUG --- status code --- 20:12:59 INFO [http_cmd]: http status code OK 20:12:59 INFO [http_cmd]: http status code OK 20:12:59 INFO [http_cmd]: http status code OK 20:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:59 INFO [http_cmd]: http status code OK 20:12:59 DEBUG --- status code --- 20:12:59 DEBUG --- status code --- 20:12:59 INFO [http_cmd]: http status code OK 20:12:59 DEBUG --- status code --- 20:12:59 DEBUG --- status code --- 20:12:59 DEBUG --- status code --- 20:12:59 INFO [http_cmd]: http status code OK 20:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:59 DEBUG --- http response --- 20:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:59 DEBUG --- status code --- 20:12:59 DEBUG --- http response --- 20:12:59 DEBUG --- http response --- 20:12:59 DEBUG --- http response --- 20:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:59 INFO [http_cmd]: http status code OK 20:12:59 DEBUG --- http response --- 20:12:59 DEBUG --- http response --- 20:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:59 INFO [http_cmd]: http status code OK 20:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:12:59 DEBUG ----- output ----- 20:12:59 DEBUG --- status code --- 20:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:59 DEBUG --- http response --- 20:12:59 DEBUG --- status code --- 20:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:59 DEBUG ----- output ----- 20:12:59 DEBUG { 20:12:59 DEBUG "status": "success", 20:12:59 DEBUG ----- output ----- 20:12:59 DEBUG ----- output ----- 20:12:59 DEBUG { 20:12:59 DEBUG "status": "success", 20:12:59 DEBUG --- http response --- 20:12:59 DEBUG ----- output ----- 20:12:59 DEBUG { 20:12:59 DEBUG { 20:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:59 DEBUG "data": { 20:12:59 DEBUG { 20:12:59 DEBUG "data": { 20:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:59 DEBUG ----- output ----- 20:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:12:59 DEBUG "status": "success", 20:12:59 DEBUG "data": { 20:12:59 DEBUG --- http response --- 20:12:59 DEBUG "resultType": "vector", 20:12:59 INFO [http_cmd]: http status code OK 20:12:59 DEBUG "status": "success", 20:12:59 DEBUG "resultType": "vector", 20:12:59 DEBUG ----- output ----- 20:12:59 DEBUG { 20:12:59 DEBUG ----- output ----- 20:12:59 DEBUG "resultType": "vector", 20:12:59 DEBUG "status": "success", 20:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:12:59 DEBUG "result": [ 20:12:59 DEBUG --- status code --- 20:12:59 DEBUG "data": { 20:12:59 DEBUG "result": [ 20:12:59 DEBUG { 20:12:59 DEBUG "status": "success", 20:12:59 DEBUG { 20:12:59 DEBUG "result": [ 20:12:59 DEBUG "data": { 20:12:59 DEBUG ----- output ----- 20:12:59 DEBUG { 20:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:12:59 DEBUG "resultType": "vector", 20:12:59 DEBUG "result": [ 20:12:59 DEBUG { 20:12:59 DEBUG "metric": { 20:12:59 DEBUG "status": "success", 20:12:59 DEBUG { 20:12:59 DEBUG "metric": { 20:12:59 DEBUG { 20:12:59 DEBUG "metric": { 20:12:59 DEBUG --- http response --- 20:12:59 DEBUG "data": { 20:12:59 DEBUG "resultType": "vector", 20:12:59 DEBUG "pod": "ds-cts-0" 20:12:59 DEBUG { 20:12:59 DEBUG "metric": { 20:12:59 DEBUG "resultType": "vector", 20:12:59 DEBUG "pod": "ds-cts-0" 20:12:59 DEBUG "status": "success", 20:12:59 DEBUG "data": { 20:12:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:12:59 DEBUG "status": "success", 20:12:59 DEBUG "result": [ 20:12:59 DEBUG }, 20:12:59 DEBUG "instance": "10.142.0.107:9100" 20:12:59 DEBUG "data": { 20:12:59 DEBUG "result": [ 20:12:59 DEBUG }, 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 20:12:59 DEBUG ----- output ----- 20:12:59 DEBUG "data": { 20:12:59 DEBUG "resultType": "vector", 20:12:59 DEBUG "value": [ 20:12:59 DEBUG }, 20:12:59 DEBUG "value": [ 20:12:59 DEBUG { 20:12:59 DEBUG 20:12:59 DEBUG "resultType": "vector", 20:12:59 DEBUG "instance": "10.142.0.107:9100" 20:12:59 DEBUG { 20:12:59 DEBUG "result": [ 20:12:59 DEBUG { 20:12:59 DEBUG "metric": { 20:12:59 DEBUG "resultType": "vector", 20:12:59 DEBUG 20:12:59 DEBUG "metric": { 20:12:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:12:59 DEBUG }, 20:12:59 DEBUG "result": [ 20:12:59 DEBUG "status": "success", 20:12:59 DEBUG { 20:12:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:59 DEBUG 20:12:59 DEBUG "result": [ 20:12:59 DEBUG { 20:12:59 DEBUG "value": [ 20:12:59 DEBUG }, 20:12:59 DEBUG { 20:12:59 DEBUG 20:12:59 DEBUG "data": { 20:12:59 DEBUG "metric": { 20:12:59 DEBUG }, 20:12:59 DEBUG 20:12:59 DEBUG "metric": { 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 20:12:59 DEBUG [...] 20:12:59 DEBUG "resultType": "vector", 20:12:59 DEBUG "instance": "10.142.0.107:9100" 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 20:12:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:12:59 DEBUG [...] 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG "result": [ 20:12:59 DEBUG }, 20:12:59 DEBUG 20:12:59 DEBUG [...] 20:12:59 DEBUG }, 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 20:12:59 DEBUG [...] 20:12:59 DEBUG "metric": { 20:12:59 DEBUG 20:12:59 DEBUG { 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG [...] 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:12:59 DEBUG 20:12:59 DEBUG "metric": { 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG }, 20:12:59 DEBUG "pod": "ds-cts-2" 20:12:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:12:59 DEBUG }, 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG "value": [ 20:12:59 DEBUG }, 20:12:59 DEBUG 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 20:12:59 DEBUG [...] 20:12:59 DEBUG "pod": "ds-cts-2" 20:12:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:59 DEBUG [...] 20:12:59 DEBUG "instance": "10.142.0.43:9100" 20:12:59 DEBUG 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG [...] 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG }, 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG "instance": "10.142.0.43:9100" 20:12:59 DEBUG }, 20:12:59 DEBUG 20:12:59 DEBUG }, 20:12:59 DEBUG 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 1713640379, 20:12:59 DEBUG "0.08726666666666667" 20:12:59 DEBUG ] 20:12:59 DEBUG 20:12:59 DEBUG }, 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 1713640379, 20:12:59 DEBUG 1713640379, 20:12:59 DEBUG [...] 20:12:59 DEBUG } 20:12:59 DEBUG "instance": "10.142.0.43:9100" 20:12:59 DEBUG [...] 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 1713640379, 20:12:59 DEBUG 20:12:59 DEBUG 20:12:59 DEBUG "0.157" 20:12:59 DEBUG "85911770.43333334" 20:12:59 DEBUG 20:12:59 DEBUG ] 20:12:59 DEBUG }, 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 1713640379, 20:12:59 DEBUG "30.466666666666665" 20:12:59 DEBUG "pod": "ds-idrepo-1" 20:12:59 DEBUG }, 20:12:59 DEBUG ] 20:12:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:12:59 DEBUG }, 20:12:59 DEBUG } 20:12:59 DEBUG } 20:12:59 DEBUG 20:12:59 DEBUG "85906470.96666667" 20:12:59 DEBUG ] 20:12:59 DEBUG "value": [ 20:12:59 DEBUG } 20:12:59 DEBUG ] 20:12:59 DEBUG 20:12:59 DEBUG "value": [ 20:12:59 DEBUG 1713640379, 20:12:59 DEBUG 20:12:59 DEBUG ] 20:12:59 DEBUG } 20:12:59 DEBUG ] 20:12:59 DEBUG 1713640379, 20:12:59 DEBUG } 20:12:59 DEBUG 20:12:59 DEBUG 1713640379, 20:12:59 DEBUG "0.0790000000000096" 20:12:59 DEBUG 20:12:59 DEBUG } 20:12:59 DEBUG } 20:12:59 DEBUG ] 20:12:59 DEBUG "0" 20:12:59 DEBUG ] 20:12:59 DEBUG "pod": "ds-idrepo-1" 20:12:59 DEBUG "962727936" 20:12:59 DEBUG ] 20:12:59 DEBUG } 20:12:59 DEBUG ] 20:12:59 DEBUG } 20:12:59 DEBUG } 20:12:59 DEBUG ] 20:12:59 DEBUG } 20:12:59 DEBUG } 20:12:59 DEBUG ] 20:12:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:12:59 DEBUG } 20:12:59 DEBUG ] 20:12:59 DEBUG } 20:12:59 DEBUG } 20:12:59 DEBUG ] 20:12:59 DEBUG } 20:12:59 DEBUG ] 20:12:59 DEBUG } 20:12:59 DEBUG } 20:12:59 DEBUG } 20:12:59 DEBUG }, 20:12:59 DEBUG }, 20:12:59 DEBUG } 20:12:59 DEBUG } 20:12:59 DEBUG "value": [ 20:12:59 DEBUG "value": [ 20:12:59 DEBUG } 20:12:59 DEBUG } 20:12:59 DEBUG 1713640379, 20:12:59 DEBUG 1713640379, 20:12:59 DEBUG "7156.452678211841" 20:12:59 DEBUG "0" 20:12:59 DEBUG ] 20:12:59 DEBUG ] 20:12:59 DEBUG } 20:12:59 DEBUG ] 20:12:59 DEBUG } 20:12:59 DEBUG } 20:12:59 DEBUG ] 20:12:59 DEBUG } 20:12:59 DEBUG } 20:12:59 DEBUG } 20:13:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640433" 20:13:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640433" 20:13:53 INFO [http_cmd]: http status code OK 20:13:53 ERROR [http_cmd]: ERROR 20:13:53 DEBUG --- status code --- 20:13:53 ERROR --- status code --- 20:13:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:13:53 DEBUG --- http response --- 20:13:53 ERROR --- http response --- 20:13:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:13:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:13:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640435" 20:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640435" 20:13:55 INFO [http_cmd]: http status code OK 20:13:55 DEBUG --- status code --- 20:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:55 DEBUG --- http response --- 20:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:55 DEBUG ----- output ----- 20:13:55 DEBUG { 20:13:55 DEBUG "status": "success", 20:13:55 DEBUG "data": { 20:13:55 DEBUG "resultType": "vector", 20:13:55 DEBUG "result": [ 20:13:55 DEBUG { 20:13:55 DEBUG "metric": { 20:13:55 DEBUG "pod": "ds-idrepo-0" 20:13:55 DEBUG }, 20:13:55 DEBUG "value": [ 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG [...] 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG "pod": "ds-idrepo-2" 20:13:55 DEBUG }, 20:13:55 DEBUG "value": [ 20:13:55 DEBUG 1713640435, 20:13:55 DEBUG "0" 20:13:55 DEBUG ] 20:13:55 DEBUG } 20:13:55 DEBUG ] 20:13:55 DEBUG } 20:13:55 DEBUG } 20:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640435" 20:13:55 INFO [http_cmd]: http status code OK 20:13:55 DEBUG --- status code --- 20:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:55 DEBUG --- http response --- 20:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:55 DEBUG ----- output ----- 20:13:55 DEBUG { 20:13:55 DEBUG "status": "success", 20:13:55 DEBUG "data": { 20:13:55 DEBUG "resultType": "vector", 20:13:55 DEBUG "result": [ 20:13:55 DEBUG { 20:13:55 DEBUG "metric": { 20:13:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:55 DEBUG }, 20:13:55 DEBUG "value": [ 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG [...] 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:55 DEBUG }, 20:13:55 DEBUG "value": [ 20:13:55 DEBUG 1713640435, 20:13:55 DEBUG "2829.733333333333" 20:13:55 DEBUG ] 20:13:55 DEBUG } 20:13:55 DEBUG ] 20:13:55 DEBUG } 20:13:55 DEBUG } 20:13:55 INFO [http_cmd]: http status code OK 20:13:55 DEBUG --- status code --- 20:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:55 DEBUG --- http response --- 20:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:55 DEBUG ----- output ----- 20:13:55 DEBUG { 20:13:55 DEBUG "status": "success", 20:13:55 DEBUG "data": { 20:13:55 DEBUG "resultType": "vector", 20:13:55 DEBUG "result": [ 20:13:55 DEBUG { 20:13:55 DEBUG "metric": { 20:13:55 DEBUG "pod": "ds-idrepo-0" 20:13:55 DEBUG }, 20:13:55 DEBUG "value": [ 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG [...] 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG "pod": "ds-idrepo-2" 20:13:55 DEBUG }, 20:13:55 DEBUG "value": [ 20:13:55 DEBUG 1713640435, 20:13:55 DEBUG "0" 20:13:55 DEBUG ] 20:13:55 DEBUG } 20:13:55 DEBUG ] 20:13:55 DEBUG } 20:13:55 DEBUG } 20:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640435" 20:13:55 INFO [http_cmd]: http status code OK 20:13:55 DEBUG --- status code --- 20:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:55 DEBUG --- http response --- 20:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:55 DEBUG ----- output ----- 20:13:55 DEBUG { 20:13:55 DEBUG "status": "success", 20:13:55 DEBUG "data": { 20:13:55 DEBUG "resultType": "vector", 20:13:55 DEBUG "result": [ 20:13:55 DEBUG { 20:13:55 DEBUG "metric": { 20:13:55 DEBUG "pod": "ds-idrepo-0" 20:13:55 DEBUG }, 20:13:55 DEBUG "value": [ 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG [...] 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:13:55 DEBUG }, 20:13:55 DEBUG "value": [ 20:13:55 DEBUG 1713640435, 20:13:55 DEBUG "0" 20:13:55 DEBUG ] 20:13:55 DEBUG } 20:13:55 DEBUG ] 20:13:55 DEBUG } 20:13:55 DEBUG } 20:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640435" 20:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640435" 20:13:55 INFO [http_cmd]: http status code OK 20:13:55 DEBUG --- status code --- 20:13:55 INFO [http_cmd]: http status code OK 20:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:55 DEBUG --- status code --- 20:13:55 DEBUG --- http response --- 20:13:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:55 DEBUG --- http response --- 20:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:55 DEBUG ----- output ----- 20:13:55 DEBUG { 20:13:55 DEBUG "status": "success", 20:13:55 DEBUG "data": { 20:13:55 DEBUG "resultType": "vector", 20:13:55 DEBUG "result": [ 20:13:55 DEBUG { 20:13:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:55 DEBUG "metric": { 20:13:55 DEBUG ----- output ----- 20:13:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:13:55 DEBUG { 20:13:55 DEBUG "status": "success", 20:13:55 DEBUG "data": { 20:13:55 DEBUG "resultType": "vector", 20:13:55 DEBUG "result": [ 20:13:55 DEBUG }, 20:13:55 DEBUG { 20:13:55 DEBUG "value": [ 20:13:55 DEBUG "metric": { 20:13:55 DEBUG 20:13:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:55 DEBUG 20:13:55 DEBUG }, 20:13:55 DEBUG "value": [ 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG [...] 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG 20:13:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:13:55 DEBUG }, 20:13:55 DEBUG "value": [ 20:13:55 DEBUG 1713640435, 20:13:55 DEBUG "9.505663001104285" 20:13:55 DEBUG 20:13:55 DEBUG ] 20:13:55 DEBUG 20:13:55 DEBUG } 20:13:55 DEBUG [...] 20:13:55 DEBUG ] 20:13:55 DEBUG 20:13:55 DEBUG } 20:13:55 DEBUG 20:13:55 DEBUG } 20:13:55 DEBUG 20:13:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:55 DEBUG }, 20:13:55 DEBUG "value": [ 20:13:55 DEBUG 1713640435, 20:13:55 DEBUG "2833.9" 20:13:55 DEBUG ] 20:13:55 DEBUG } 20:13:55 DEBUG ] 20:13:55 DEBUG } 20:13:55 DEBUG } 20:13:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640435" 20:13:56 INFO [http_cmd]: http status code OK 20:13:56 DEBUG --- status code --- 20:13:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:56 DEBUG --- http response --- 20:13:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:56 DEBUG ----- output ----- 20:13:56 DEBUG { 20:13:56 DEBUG "status": "success", 20:13:56 DEBUG "data": { 20:13:56 DEBUG "resultType": "vector", 20:13:56 DEBUG "result": [ 20:13:56 DEBUG { 20:13:56 DEBUG "metric": { 20:13:56 DEBUG "instance": "10.142.0.107:9100" 20:13:56 DEBUG }, 20:13:56 DEBUG "value": [ 20:13:56 DEBUG 20:13:56 DEBUG 20:13:56 DEBUG 20:13:56 DEBUG [...] 20:13:56 DEBUG 20:13:56 DEBUG 20:13:56 DEBUG 20:13:56 DEBUG "instance": "10.142.0.43:9100" 20:13:56 DEBUG }, 20:13:56 DEBUG "value": [ 20:13:56 DEBUG 1713640435, 20:13:56 DEBUG "3703119213568" 20:13:56 DEBUG ] 20:13:56 DEBUG } 20:13:56 DEBUG ] 20:13:56 DEBUG } 20:13:56 DEBUG } 20:13:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640436" 20:13:56 INFO [http_cmd]: http status code OK 20:13:56 DEBUG --- status code --- 20:13:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:56 DEBUG --- http response --- 20:13:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:56 DEBUG ----- output ----- 20:13:56 DEBUG { 20:13:56 DEBUG "status": "success", 20:13:56 DEBUG "data": { 20:13:56 DEBUG "resultType": "vector", 20:13:56 DEBUG "result": [ 20:13:56 DEBUG { 20:13:56 DEBUG "metric": { 20:13:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:56 DEBUG }, 20:13:56 DEBUG "value": [ 20:13:56 DEBUG 20:13:56 DEBUG 20:13:56 DEBUG 20:13:56 DEBUG [...] 20:13:56 DEBUG 20:13:56 DEBUG 20:13:56 DEBUG 20:13:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:56 DEBUG }, 20:13:56 DEBUG "value": [ 20:13:56 DEBUG 1713640436, 20:13:56 DEBUG "2834.5" 20:13:56 DEBUG ] 20:13:56 DEBUG } 20:13:56 DEBUG ] 20:13:56 DEBUG } 20:13:56 DEBUG } 20:13:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640436" 20:13:56 INFO [http_cmd]: http status code OK 20:13:56 DEBUG --- status code --- 20:13:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:56 DEBUG --- http response --- 20:13:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:56 DEBUG ----- output ----- 20:13:56 DEBUG { 20:13:56 DEBUG "status": "success", 20:13:56 DEBUG "data": { 20:13:56 DEBUG "resultType": "vector", 20:13:56 DEBUG "result": [ 20:13:56 DEBUG { 20:13:56 DEBUG "metric": { 20:13:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:56 DEBUG }, 20:13:56 DEBUG "value": [ 20:13:56 DEBUG 20:13:56 DEBUG 20:13:56 DEBUG 20:13:56 DEBUG [...] 20:13:56 DEBUG 20:13:56 DEBUG 20:13:56 DEBUG 20:13:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:56 DEBUG }, 20:13:56 DEBUG "value": [ 20:13:56 DEBUG 1713640436, 20:13:56 DEBUG "2834.4666666666667" 20:13:56 DEBUG ] 20:13:56 DEBUG } 20:13:56 DEBUG ] 20:13:56 DEBUG } 20:13:56 DEBUG } 20:13:56 INFO 20:13:56 INFO [loop_until]: kubectl --namespace=xlou top pods 20:13:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:13:57 INFO [loop_until]: OK (rc = 0) 20:13:57 DEBUG --- stdout --- 20:13:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18420m 8657Mi am-767f6b67b5-qcqz2 18556m 9070Mi am-767f6b67b5-vbltz 19993m 8675Mi ds-cts-0 19506m 10964Mi ds-cts-1 18275m 10774Mi ds-cts-2 20362m 11058Mi ds-idrepo-0 7964m 3893Mi ds-idrepo-1 7881m 3882Mi ds-idrepo-2 8113m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9566m 824Mi 20:13:57 DEBUG --- stderr --- 20:13:57 DEBUG 20:13:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640437" 20:13:57 INFO [http_cmd]: http status code OK 20:13:57 DEBUG --- status code --- 20:13:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:57 DEBUG --- http response --- 20:13:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:13:57 DEBUG ----- output ----- 20:13:57 DEBUG { 20:13:57 DEBUG "status": "success", 20:13:57 DEBUG "data": { 20:13:57 DEBUG "resultType": "vector", 20:13:57 DEBUG "result": [ 20:13:57 DEBUG { 20:13:57 DEBUG "metric": { 20:13:57 DEBUG "pod": "ds-idrepo-0" 20:13:57 DEBUG }, 20:13:57 DEBUG "value": [ 20:13:57 DEBUG 20:13:57 DEBUG 20:13:57 DEBUG 20:13:57 DEBUG [...] 20:13:57 DEBUG 20:13:57 DEBUG 20:13:57 DEBUG 20:13:57 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:13:57 DEBUG }, 20:13:57 DEBUG "value": [ 20:13:57 DEBUG 1713640437, 20:13:57 DEBUG "0" 20:13:57 DEBUG ] 20:13:57 DEBUG } 20:13:57 DEBUG ] 20:13:57 DEBUG } 20:13:57 DEBUG } 20:13:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640437" 20:13:57 INFO [http_cmd]: http status code OK 20:13:57 DEBUG --- status code --- 20:13:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:57 DEBUG --- http response --- 20:13:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:57 DEBUG ----- output ----- 20:13:57 DEBUG { 20:13:57 DEBUG "status": "success", 20:13:57 DEBUG "data": { 20:13:57 DEBUG "resultType": "vector", 20:13:57 DEBUG "result": [ 20:13:57 DEBUG { 20:13:57 DEBUG "metric": { 20:13:57 DEBUG "pod": "ds-idrepo-0" 20:13:57 DEBUG }, 20:13:57 DEBUG "value": [ 20:13:57 DEBUG 20:13:57 DEBUG 20:13:57 DEBUG 20:13:57 DEBUG [...] 20:13:57 DEBUG 20:13:57 DEBUG 20:13:57 DEBUG 20:13:57 DEBUG "pod": "ds-idrepo-2" 20:13:57 DEBUG }, 20:13:57 DEBUG "value": [ 20:13:57 DEBUG 1713640437, 20:13:57 DEBUG "0" 20:13:57 DEBUG ] 20:13:57 DEBUG } 20:13:57 DEBUG ] 20:13:57 DEBUG } 20:13:57 DEBUG } 20:13:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640437" 20:13:57 INFO [http_cmd]: http status code OK 20:13:57 DEBUG --- status code --- 20:13:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:57 DEBUG --- http response --- 20:13:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:13:57 DEBUG ----- output ----- 20:13:57 DEBUG { 20:13:57 DEBUG "status": "success", 20:13:57 DEBUG "data": { 20:13:57 DEBUG "resultType": "vector", 20:13:57 DEBUG "result": [ 20:13:57 DEBUG { 20:13:57 DEBUG "metric": { 20:13:57 DEBUG "pod": "ds-cts-0" 20:13:57 DEBUG }, 20:13:57 DEBUG "value": [ 20:13:57 DEBUG 20:13:57 DEBUG 20:13:57 DEBUG 20:13:57 DEBUG [...] 20:13:57 DEBUG 20:13:57 DEBUG 20:13:57 DEBUG 20:13:57 DEBUG "pod": "ds-cts-2" 20:13:57 DEBUG }, 20:13:57 DEBUG "value": [ 20:13:57 DEBUG 1713640437, 20:13:57 DEBUG "0" 20:13:57 DEBUG ] 20:13:57 DEBUG } 20:13:57 DEBUG ] 20:13:57 DEBUG } 20:13:57 DEBUG } 20:13:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640438" 20:13:58 INFO [http_cmd]: http status code OK 20:13:58 DEBUG --- status code --- 20:13:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:58 DEBUG --- http response --- 20:13:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:13:58 DEBUG ----- output ----- 20:13:58 DEBUG { 20:13:58 DEBUG "status": "success", 20:13:58 DEBUG "data": { 20:13:58 DEBUG "resultType": "vector", 20:13:58 DEBUG "result": [ 20:13:58 DEBUG { 20:13:58 DEBUG "metric": { 20:13:58 DEBUG "pod": "ds-idrepo-0" 20:13:58 DEBUG }, 20:13:58 DEBUG "value": [ 20:13:58 DEBUG 20:13:58 DEBUG 20:13:58 DEBUG 20:13:58 DEBUG [...] 20:13:58 DEBUG 20:13:58 DEBUG 20:13:58 DEBUG 20:13:58 DEBUG "pod": "ds-cts-2" 20:13:58 DEBUG }, 20:13:58 DEBUG "value": [ 20:13:58 DEBUG 1713640438, 20:13:58 DEBUG "0" 20:13:58 DEBUG ] 20:13:58 DEBUG } 20:13:58 DEBUG ] 20:13:58 DEBUG } 20:13:58 DEBUG } 20:13:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640438" 20:13:58 INFO [http_cmd]: http status code OK 20:13:58 DEBUG --- status code --- 20:13:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:58 DEBUG --- http response --- 20:13:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:58 DEBUG ----- output ----- 20:13:58 DEBUG { 20:13:58 DEBUG "status": "success", 20:13:58 DEBUG "data": { 20:13:58 DEBUG "resultType": "vector", 20:13:58 DEBUG "result": [ 20:13:58 DEBUG { 20:13:58 DEBUG "metric": { 20:13:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:58 DEBUG }, 20:13:58 DEBUG "value": [ 20:13:58 DEBUG 20:13:58 DEBUG 20:13:58 DEBUG 20:13:58 DEBUG [...] 20:13:58 DEBUG 20:13:58 DEBUG 20:13:58 DEBUG 20:13:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:58 DEBUG }, 20:13:58 DEBUG "value": [ 20:13:58 DEBUG 1713640438, 20:13:58 DEBUG "509987" 20:13:58 DEBUG ] 20:13:58 DEBUG } 20:13:58 DEBUG ] 20:13:58 DEBUG } 20:13:58 DEBUG } 20:13:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640438" 20:13:58 INFO 20:13:58 INFO [loop_until]: kubectl --namespace=xlou top node 20:13:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:13:58 INFO [http_cmd]: http status code OK 20:13:58 DEBUG --- status code --- 20:13:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:58 DEBUG --- http response --- 20:13:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:58 DEBUG ----- output ----- 20:13:58 DEBUG { 20:13:58 DEBUG "status": "success", 20:13:58 DEBUG "data": { 20:13:58 DEBUG "resultType": "vector", 20:13:58 DEBUG "result": [ 20:13:58 DEBUG { 20:13:58 DEBUG "metric": { 20:13:58 DEBUG "instance": "10.142.0.107:9100" 20:13:58 DEBUG }, 20:13:58 DEBUG "value": [ 20:13:58 DEBUG 20:13:58 DEBUG 20:13:58 DEBUG 20:13:58 DEBUG [...] 20:13:58 DEBUG 20:13:58 DEBUG 20:13:58 DEBUG 20:13:58 DEBUG "instance": "10.142.0.43:9100" 20:13:58 DEBUG }, 20:13:58 DEBUG "value": [ 20:13:58 DEBUG 1713640438, 20:13:58 DEBUG "0.2507000000000337" 20:13:58 DEBUG ] 20:13:58 DEBUG } 20:13:58 DEBUG ] 20:13:58 DEBUG } 20:13:58 DEBUG } 20:13:58 INFO [loop_until]: OK (rc = 0) 20:13:58 DEBUG --- stdout --- 20:13:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6212Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8201m 27% 7647Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18287m 61% 11374Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3348Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19833m 66% 14166Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5185Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18528m 62% 11953Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15693m 52% 2975Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20940m 70% 10883Mi 9% gke-xlou-cdm-ds-0f601751-j488 8745m 29% 7218Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19040m 63% 15816Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8600m 28% 7866Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20438m 68% 13704Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9786m 61% 6399Mi 10% 20:13:58 DEBUG --- stderr --- 20:13:58 DEBUG 20:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640439" 20:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640439" 20:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640439" 20:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640439" 20:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640439" 20:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640439" 20:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640439" 20:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640439" 20:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640439" 20:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640439" 20:13:59 INFO [http_cmd]: http status code OK 20:13:59 INFO [http_cmd]: http status code OK 20:13:59 INFO [http_cmd]: http status code OK 20:13:59 DEBUG --- status code --- 20:13:59 INFO [http_cmd]: http status code OK 20:13:59 INFO [http_cmd]: http status code OK 20:13:59 DEBUG --- status code --- 20:13:59 DEBUG --- status code --- 20:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:59 DEBUG --- status code --- 20:13:59 DEBUG --- status code --- 20:13:59 INFO [http_cmd]: http status code OK 20:13:59 INFO [http_cmd]: http status code OK 20:13:59 INFO [http_cmd]: http status code OK 20:13:59 INFO [http_cmd]: http status code OK 20:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:59 DEBUG --- http response --- 20:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:59 DEBUG --- status code --- 20:13:59 DEBUG --- status code --- 20:13:59 DEBUG --- status code --- 20:13:59 DEBUG --- status code --- 20:13:59 DEBUG --- http response --- 20:13:59 DEBUG --- http response --- 20:13:59 INFO [http_cmd]: http status code OK 20:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:13:59 DEBUG --- http response --- 20:13:59 DEBUG --- http response --- 20:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:13:59 DEBUG --- status code --- 20:13:59 DEBUG ----- output ----- 20:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:13:59 DEBUG --- http response --- 20:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:59 DEBUG --- http response --- 20:13:59 DEBUG --- http response --- 20:13:59 DEBUG --- http response --- 20:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:59 DEBUG ----- output ----- 20:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:13:59 DEBUG { 20:13:59 DEBUG ----- output ----- 20:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:59 DEBUG ----- output ----- 20:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:13:59 DEBUG ----- output ----- 20:13:59 DEBUG ----- output ----- 20:13:59 DEBUG { 20:13:59 DEBUG "status": "success", 20:13:59 DEBUG "status": "success", 20:13:59 DEBUG { 20:13:59 DEBUG "status": "success", 20:13:59 DEBUG { 20:13:59 DEBUG ----- output ----- 20:13:59 DEBUG ----- output ----- 20:13:59 DEBUG { 20:13:59 DEBUG { 20:13:59 DEBUG "data": { 20:13:59 DEBUG --- http response --- 20:13:59 DEBUG "data": { 20:13:59 DEBUG "data": { 20:13:59 DEBUG ----- output ----- 20:13:59 DEBUG "status": "success", 20:13:59 DEBUG { 20:13:59 DEBUG { 20:13:59 DEBUG "status": "success", 20:13:59 DEBUG "status": "success", 20:13:59 DEBUG "resultType": "vector", 20:13:59 DEBUG "resultType": "vector", 20:13:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:13:59 DEBUG "resultType": "vector", 20:13:59 DEBUG { 20:13:59 DEBUG "data": { 20:13:59 DEBUG "status": "success", 20:13:59 DEBUG "status": "success", 20:13:59 DEBUG "data": { 20:13:59 DEBUG "data": { 20:13:59 DEBUG "result": [ 20:13:59 DEBUG "result": [ 20:13:59 DEBUG ----- output ----- 20:13:59 DEBUG "result": [ 20:13:59 DEBUG "status": "success", 20:13:59 DEBUG "resultType": "vector", 20:13:59 DEBUG "data": { 20:13:59 DEBUG "data": { 20:13:59 DEBUG "resultType": "vector", 20:13:59 DEBUG "resultType": "vector", 20:13:59 DEBUG { 20:13:59 DEBUG { 20:13:59 DEBUG { 20:13:59 DEBUG { 20:13:59 DEBUG "data": { 20:13:59 DEBUG "result": [ 20:13:59 DEBUG "resultType": "vector", 20:13:59 DEBUG "resultType": "vector", 20:13:59 DEBUG "result": [ 20:13:59 DEBUG "result": [ 20:13:59 DEBUG "metric": { 20:13:59 DEBUG "metric": { 20:13:59 DEBUG "status": "success", 20:13:59 DEBUG "metric": { 20:13:59 DEBUG "resultType": "vector", 20:13:59 DEBUG { 20:13:59 DEBUG "result": [ 20:13:59 DEBUG "result": [ 20:13:59 DEBUG { 20:13:59 DEBUG { 20:13:59 DEBUG "instance": "10.142.0.107:9100" 20:13:59 DEBUG "pod": "ds-cts-0" 20:13:59 DEBUG "data": { 20:13:59 DEBUG "pod": "ds-idrepo-0" 20:13:59 DEBUG "result": [ 20:13:59 DEBUG "metric": { 20:13:59 DEBUG { 20:13:59 DEBUG { 20:13:59 DEBUG "metric": { 20:13:59 DEBUG "metric": { 20:13:59 DEBUG }, 20:13:59 DEBUG }, 20:13:59 DEBUG "value": [ 20:13:59 DEBUG "resultType": "vector", 20:13:59 DEBUG { 20:13:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:59 DEBUG "metric": { 20:13:59 DEBUG "metric": { 20:13:59 DEBUG "instance": "10.142.0.107:9100" 20:13:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:13:59 DEBUG "value": [ 20:13:59 DEBUG 20:13:59 DEBUG }, 20:13:59 DEBUG "result": [ 20:13:59 DEBUG "metric": { 20:13:59 DEBUG }, 20:13:59 DEBUG "pod": "ds-idrepo-0" 20:13:59 DEBUG "pod": "ds-cts-0" 20:13:59 DEBUG }, 20:13:59 DEBUG }, 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG "value": [ 20:13:59 DEBUG 20:13:59 DEBUG "instance": "10.142.0.107:9100" 20:13:59 DEBUG "value": [ 20:13:59 DEBUG }, 20:13:59 DEBUG }, 20:13:59 DEBUG "value": [ 20:13:59 DEBUG "value": [ 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG { 20:13:59 DEBUG 20:13:59 DEBUG }, 20:13:59 DEBUG 20:13:59 DEBUG "value": [ 20:13:59 DEBUG "value": [ 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG [...] 20:13:59 DEBUG "metric": { 20:13:59 DEBUG 20:13:59 DEBUG "value": [ 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG [...] 20:13:59 DEBUG 20:13:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:13:59 DEBUG [...] 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG }, 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG [...] 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG [...] 20:13:59 DEBUG [...] 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG "value": [ 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG [...] 20:13:59 DEBUG [...] 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG "pod": "ds-cts-2" 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG [...] 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG "instance": "10.142.0.43:9100" 20:13:59 DEBUG }, 20:13:59 DEBUG 20:13:59 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:13:59 DEBUG }, 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG }, 20:13:59 DEBUG "value": [ 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG "value": [ 20:13:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG "instance": "10.142.0.43:9100" 20:13:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:13:59 DEBUG "value": [ 20:13:59 DEBUG 1713640439, 20:13:59 DEBUG [...] 20:13:59 DEBUG 20:13:59 DEBUG 1713640439, 20:13:59 DEBUG }, 20:13:59 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:13:59 DEBUG "pod": "ds-cts-2" 20:13:59 DEBUG }, 20:13:59 DEBUG }, 20:13:59 DEBUG 1713640439, 20:13:59 DEBUG "0.07336666666666666" 20:13:59 DEBUG 20:13:59 DEBUG 20:13:59 DEBUG "0" 20:13:59 DEBUG "value": [ 20:13:59 DEBUG }, 20:13:59 DEBUG }, 20:13:59 DEBUG "value": [ 20:13:59 DEBUG "value": [ 20:13:59 DEBUG "85853884.69999999" 20:13:59 DEBUG ] 20:13:59 DEBUG 20:13:59 DEBUG "instance": "10.142.0.43:9100" 20:13:59 DEBUG ] 20:13:59 DEBUG 1713640439, 20:13:59 DEBUG "value": [ 20:13:59 DEBUG "value": [ 20:13:59 DEBUG 1713640439, 20:13:59 DEBUG 1713640439, 20:13:59 DEBUG ] 20:13:59 DEBUG } 20:13:59 DEBUG 20:13:59 DEBUG }, 20:13:59 DEBUG } 20:13:59 DEBUG "0.157" 20:13:59 DEBUG 1713640439, 20:13:59 DEBUG 1713640439, 20:13:59 DEBUG "85849848.26666667" 20:13:59 DEBUG "964259840" 20:13:59 DEBUG ] 20:13:59 DEBUG ] 20:13:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:13:59 DEBUG "value": [ 20:13:59 DEBUG 1713640439, 20:13:59 DEBUG ] 20:13:59 DEBUG "0" 20:13:59 DEBUG "29.266666666666666" 20:13:59 DEBUG ] 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG }, 20:13:59 DEBUG "0.08099999999999453" 20:13:59 DEBUG ] 20:13:59 DEBUG } 20:13:59 DEBUG ] 20:13:59 DEBUG ] 20:13:59 DEBUG } 20:13:59 DEBUG ] 20:13:59 DEBUG ] 20:13:59 DEBUG } 20:13:59 DEBUG "value": [ 20:13:59 DEBUG ] 20:13:59 DEBUG } 20:13:59 DEBUG ] 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG ] 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG 1713640439, 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG ] 20:13:59 DEBUG ] 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG "0" 20:13:59 DEBUG ] 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG ] 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG } 20:13:59 DEBUG ] 20:13:59 DEBUG } 20:13:59 DEBUG } 20:14:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640493" 20:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640494" 20:14:54 ERROR [http_cmd]: ERROR 20:14:54 INFO [http_cmd]: http status code OK 20:14:54 ERROR --- status code --- 20:14:54 DEBUG --- status code --- 20:14:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:54 ERROR --- http response --- 20:14:54 DEBUG --- http response --- 20:14:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:14:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:14:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640495" 20:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640495" 20:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640495" 20:14:55 INFO [http_cmd]: http status code OK 20:14:55 DEBUG --- status code --- 20:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:55 DEBUG --- http response --- 20:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:55 DEBUG ----- output ----- 20:14:55 DEBUG { 20:14:55 DEBUG "status": "success", 20:14:55 DEBUG "data": { 20:14:55 DEBUG "resultType": "vector", 20:14:55 DEBUG "result": [ 20:14:55 DEBUG { 20:14:55 DEBUG "metric": { 20:14:55 DEBUG "pod": "ds-idrepo-0" 20:14:55 DEBUG }, 20:14:55 DEBUG "value": [ 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG [...] 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG "pod": "ds-idrepo-2" 20:14:55 DEBUG }, 20:14:55 DEBUG "value": [ 20:14:55 DEBUG 1713640495, 20:14:55 DEBUG "0" 20:14:55 DEBUG ] 20:14:55 DEBUG } 20:14:55 DEBUG ] 20:14:55 DEBUG } 20:14:55 DEBUG } 20:14:55 INFO [http_cmd]: http status code OK 20:14:55 DEBUG --- status code --- 20:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:55 DEBUG --- http response --- 20:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:55 DEBUG ----- output ----- 20:14:55 DEBUG { 20:14:55 DEBUG "status": "success", 20:14:55 DEBUG "data": { 20:14:55 DEBUG "resultType": "vector", 20:14:55 DEBUG "result": [ 20:14:55 DEBUG { 20:14:55 DEBUG "metric": { 20:14:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:55 DEBUG }, 20:14:55 DEBUG "value": [ 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG [...] 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:55 DEBUG }, 20:14:55 DEBUG "value": [ 20:14:55 DEBUG 1713640495, 20:14:55 DEBUG "2820.2666666666664" 20:14:55 DEBUG ] 20:14:55 DEBUG } 20:14:55 DEBUG ] 20:14:55 DEBUG } 20:14:55 DEBUG } 20:14:55 INFO [http_cmd]: http status code OK 20:14:55 DEBUG --- status code --- 20:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:55 DEBUG --- http response --- 20:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:55 DEBUG ----- output ----- 20:14:55 DEBUG { 20:14:55 DEBUG "status": "success", 20:14:55 DEBUG "data": { 20:14:55 DEBUG "resultType": "vector", 20:14:55 DEBUG "result": [ 20:14:55 DEBUG { 20:14:55 DEBUG "metric": { 20:14:55 DEBUG "pod": "ds-idrepo-0" 20:14:55 DEBUG }, 20:14:55 DEBUG "value": [ 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG [...] 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG "pod": "ds-idrepo-2" 20:14:55 DEBUG }, 20:14:55 DEBUG "value": [ 20:14:55 DEBUG 1713640495, 20:14:55 DEBUG "0" 20:14:55 DEBUG ] 20:14:55 DEBUG } 20:14:55 DEBUG ] 20:14:55 DEBUG } 20:14:55 DEBUG } 20:14:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640495" 20:14:55 INFO [http_cmd]: http status code OK 20:14:55 DEBUG --- status code --- 20:14:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:55 DEBUG --- http response --- 20:14:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:55 DEBUG ----- output ----- 20:14:55 DEBUG { 20:14:55 DEBUG "status": "success", 20:14:55 DEBUG "data": { 20:14:55 DEBUG "resultType": "vector", 20:14:55 DEBUG "result": [ 20:14:55 DEBUG { 20:14:55 DEBUG "metric": { 20:14:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:14:55 DEBUG }, 20:14:55 DEBUG "value": [ 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG [...] 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG 20:14:55 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:14:55 DEBUG }, 20:14:55 DEBUG "value": [ 20:14:55 DEBUG 1713640495, 20:14:55 DEBUG "0" 20:14:55 DEBUG ] 20:14:55 DEBUG } 20:14:55 DEBUG ] 20:14:55 DEBUG } 20:14:55 DEBUG } 20:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640496" 20:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640496" 20:14:56 INFO [http_cmd]: http status code OK 20:14:56 DEBUG --- status code --- 20:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:56 DEBUG --- http response --- 20:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:56 DEBUG ----- output ----- 20:14:56 INFO [http_cmd]: http status code OK 20:14:56 DEBUG { 20:14:56 DEBUG --- status code --- 20:14:56 DEBUG "status": "success", 20:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:56 DEBUG "data": { 20:14:56 DEBUG --- http response --- 20:14:56 DEBUG "resultType": "vector", 20:14:56 DEBUG "result": [ 20:14:56 DEBUG { 20:14:56 DEBUG "metric": { 20:14:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:56 DEBUG }, 20:14:56 DEBUG "value": [ 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG [...] 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:56 DEBUG }, 20:14:56 DEBUG "value": [ 20:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:56 DEBUG 1713640496, 20:14:56 DEBUG ----- output ----- 20:14:56 DEBUG "2833.366666666667" 20:14:56 DEBUG { 20:14:56 DEBUG ] 20:14:56 DEBUG "status": "success", 20:14:56 DEBUG } 20:14:56 DEBUG "data": { 20:14:56 DEBUG ] 20:14:56 DEBUG "resultType": "vector", 20:14:56 DEBUG } 20:14:56 DEBUG "result": [ 20:14:56 DEBUG } 20:14:56 DEBUG { 20:14:56 DEBUG "metric": { 20:14:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:14:56 DEBUG }, 20:14:56 DEBUG "value": [ 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG [...] 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:14:56 DEBUG }, 20:14:56 DEBUG "value": [ 20:14:56 DEBUG 1713640496, 20:14:56 DEBUG "9.597821515830164" 20:14:56 DEBUG ] 20:14:56 DEBUG } 20:14:56 DEBUG ] 20:14:56 DEBUG } 20:14:56 DEBUG } 20:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640496" 20:14:56 INFO [http_cmd]: http status code OK 20:14:56 DEBUG --- status code --- 20:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:56 DEBUG --- http response --- 20:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:56 DEBUG ----- output ----- 20:14:56 DEBUG { 20:14:56 DEBUG "status": "success", 20:14:56 DEBUG "data": { 20:14:56 DEBUG "resultType": "vector", 20:14:56 DEBUG "result": [ 20:14:56 DEBUG { 20:14:56 DEBUG "metric": { 20:14:56 DEBUG "instance": "10.142.0.107:9100" 20:14:56 DEBUG }, 20:14:56 DEBUG "value": [ 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG [...] 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG "instance": "10.142.0.43:9100" 20:14:56 DEBUG }, 20:14:56 DEBUG "value": [ 20:14:56 DEBUG 1713640496, 20:14:56 DEBUG "3708047539200" 20:14:56 DEBUG ] 20:14:56 DEBUG } 20:14:56 DEBUG ] 20:14:56 DEBUG } 20:14:56 DEBUG } 20:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640496" 20:14:56 INFO [http_cmd]: http status code OK 20:14:56 DEBUG --- status code --- 20:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:56 DEBUG --- http response --- 20:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:56 DEBUG ----- output ----- 20:14:56 DEBUG { 20:14:56 DEBUG "status": "success", 20:14:56 DEBUG "data": { 20:14:56 DEBUG "resultType": "vector", 20:14:56 DEBUG "result": [ 20:14:56 DEBUG { 20:14:56 DEBUG "metric": { 20:14:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:56 DEBUG }, 20:14:56 DEBUG "value": [ 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG [...] 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:56 DEBUG }, 20:14:56 DEBUG "value": [ 20:14:56 DEBUG 1713640496, 20:14:56 DEBUG "2832.7666666666664" 20:14:56 DEBUG ] 20:14:56 DEBUG } 20:14:56 DEBUG ] 20:14:56 DEBUG } 20:14:56 DEBUG } 20:14:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640496" 20:14:56 INFO [http_cmd]: http status code OK 20:14:56 DEBUG --- status code --- 20:14:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:56 DEBUG --- http response --- 20:14:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:56 DEBUG ----- output ----- 20:14:56 DEBUG { 20:14:56 DEBUG "status": "success", 20:14:56 DEBUG "data": { 20:14:56 DEBUG "resultType": "vector", 20:14:56 DEBUG "result": [ 20:14:56 DEBUG { 20:14:56 DEBUG "metric": { 20:14:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:56 DEBUG }, 20:14:56 DEBUG "value": [ 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG [...] 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG 20:14:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:56 DEBUG }, 20:14:56 DEBUG "value": [ 20:14:56 DEBUG 1713640496, 20:14:56 DEBUG "2833.233333333333" 20:14:56 DEBUG ] 20:14:56 DEBUG } 20:14:56 DEBUG ] 20:14:56 DEBUG } 20:14:56 DEBUG } 20:14:57 INFO 20:14:57 INFO [loop_until]: kubectl --namespace=xlou top pods 20:14:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:14:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640497" 20:14:57 INFO [http_cmd]: http status code OK 20:14:57 DEBUG --- status code --- 20:14:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:57 DEBUG --- http response --- 20:14:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:14:57 DEBUG ----- output ----- 20:14:57 DEBUG { 20:14:57 DEBUG "status": "success", 20:14:57 DEBUG "data": { 20:14:57 DEBUG "resultType": "vector", 20:14:57 DEBUG "result": [ 20:14:57 DEBUG { 20:14:57 DEBUG "metric": { 20:14:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:14:57 DEBUG }, 20:14:57 DEBUG "value": [ 20:14:57 DEBUG 20:14:57 DEBUG 20:14:57 DEBUG 20:14:57 DEBUG [...] 20:14:57 DEBUG 20:14:57 DEBUG 20:14:57 DEBUG 20:14:57 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:14:57 DEBUG }, 20:14:57 DEBUG "value": [ 20:14:57 DEBUG 1713640497, 20:14:57 DEBUG "0" 20:14:57 DEBUG ] 20:14:57 DEBUG } 20:14:57 DEBUG ] 20:14:57 DEBUG } 20:14:57 DEBUG } 20:14:57 INFO [loop_until]: OK (rc = 0) 20:14:57 DEBUG --- stdout --- 20:14:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18279m 8657Mi am-767f6b67b5-qcqz2 18676m 9076Mi am-767f6b67b5-vbltz 20041m 8674Mi ds-cts-0 19893m 10981Mi ds-cts-1 18598m 10776Mi ds-cts-2 20434m 11052Mi ds-idrepo-0 8057m 3893Mi ds-idrepo-1 7869m 3882Mi ds-idrepo-2 7927m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9575m 827Mi 20:14:57 DEBUG --- stderr --- 20:14:57 DEBUG 20:14:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640497" 20:14:57 INFO [http_cmd]: http status code OK 20:14:57 DEBUG --- status code --- 20:14:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:57 DEBUG --- http response --- 20:14:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:57 DEBUG ----- output ----- 20:14:57 DEBUG { 20:14:57 DEBUG "status": "success", 20:14:57 DEBUG "data": { 20:14:57 DEBUG "resultType": "vector", 20:14:57 DEBUG "result": [ 20:14:57 DEBUG { 20:14:57 DEBUG "metric": { 20:14:57 DEBUG "pod": "ds-idrepo-0" 20:14:57 DEBUG }, 20:14:57 DEBUG "value": [ 20:14:57 DEBUG 20:14:57 DEBUG 20:14:57 DEBUG 20:14:57 DEBUG [...] 20:14:57 DEBUG 20:14:57 DEBUG 20:14:57 DEBUG 20:14:57 DEBUG "pod": "ds-idrepo-2" 20:14:57 DEBUG }, 20:14:57 DEBUG "value": [ 20:14:57 DEBUG 1713640497, 20:14:57 DEBUG "0" 20:14:57 DEBUG ] 20:14:57 DEBUG } 20:14:57 DEBUG ] 20:14:57 DEBUG } 20:14:57 DEBUG } 20:14:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640497" 20:14:57 INFO [http_cmd]: http status code OK 20:14:57 DEBUG --- status code --- 20:14:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:57 DEBUG --- http response --- 20:14:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:14:57 DEBUG ----- output ----- 20:14:57 DEBUG { 20:14:57 DEBUG "status": "success", 20:14:57 DEBUG "data": { 20:14:57 DEBUG "resultType": "vector", 20:14:57 DEBUG "result": [ 20:14:57 DEBUG { 20:14:57 DEBUG "metric": { 20:14:57 DEBUG "pod": "ds-cts-0" 20:14:57 DEBUG }, 20:14:57 DEBUG "value": [ 20:14:57 DEBUG 20:14:57 DEBUG 20:14:57 DEBUG 20:14:57 DEBUG [...] 20:14:57 DEBUG 20:14:57 DEBUG 20:14:57 DEBUG 20:14:57 DEBUG "pod": "ds-cts-2" 20:14:57 DEBUG }, 20:14:57 DEBUG "value": [ 20:14:57 DEBUG 1713640497, 20:14:57 DEBUG "0" 20:14:57 DEBUG ] 20:14:57 DEBUG } 20:14:57 DEBUG ] 20:14:57 DEBUG } 20:14:57 DEBUG } 20:14:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640498" 20:14:58 INFO [http_cmd]: http status code OK 20:14:58 DEBUG --- status code --- 20:14:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:58 DEBUG --- http response --- 20:14:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:14:58 DEBUG ----- output ----- 20:14:58 DEBUG { 20:14:58 DEBUG "status": "success", 20:14:58 DEBUG "data": { 20:14:58 DEBUG "resultType": "vector", 20:14:58 DEBUG "result": [ 20:14:58 DEBUG { 20:14:58 DEBUG "metric": { 20:14:58 DEBUG "pod": "ds-idrepo-0" 20:14:58 DEBUG }, 20:14:58 DEBUG "value": [ 20:14:58 DEBUG 20:14:58 DEBUG 20:14:58 DEBUG 20:14:58 DEBUG [...] 20:14:58 DEBUG 20:14:58 DEBUG 20:14:58 DEBUG 20:14:58 DEBUG "pod": "ds-cts-2" 20:14:58 DEBUG }, 20:14:58 DEBUG "value": [ 20:14:58 DEBUG 1713640498, 20:14:58 DEBUG "0" 20:14:58 DEBUG ] 20:14:58 DEBUG } 20:14:58 DEBUG ] 20:14:58 DEBUG } 20:14:58 DEBUG } 20:14:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640498" 20:14:58 INFO [http_cmd]: http status code OK 20:14:58 DEBUG --- status code --- 20:14:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:58 DEBUG --- http response --- 20:14:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:58 DEBUG ----- output ----- 20:14:58 DEBUG { 20:14:58 DEBUG "status": "success", 20:14:58 DEBUG "data": { 20:14:58 DEBUG "resultType": "vector", 20:14:58 DEBUG "result": [ 20:14:58 DEBUG { 20:14:58 DEBUG "metric": { 20:14:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:58 DEBUG }, 20:14:58 DEBUG "value": [ 20:14:58 DEBUG 20:14:58 DEBUG 20:14:58 DEBUG 20:14:58 DEBUG [...] 20:14:58 DEBUG 20:14:58 DEBUG 20:14:58 DEBUG 20:14:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:58 DEBUG }, 20:14:58 DEBUG "value": [ 20:14:58 DEBUG 1713640498, 20:14:58 DEBUG "510086" 20:14:58 DEBUG ] 20:14:58 DEBUG } 20:14:58 DEBUG ] 20:14:58 DEBUG } 20:14:58 DEBUG } 20:14:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640498" 20:14:58 INFO [http_cmd]: http status code OK 20:14:58 DEBUG --- status code --- 20:14:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:58 DEBUG --- http response --- 20:14:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:58 DEBUG ----- output ----- 20:14:58 DEBUG { 20:14:58 DEBUG "status": "success", 20:14:58 DEBUG "data": { 20:14:58 DEBUG "resultType": "vector", 20:14:58 DEBUG "result": [ 20:14:58 DEBUG { 20:14:58 DEBUG "metric": { 20:14:58 DEBUG "instance": "10.142.0.107:9100" 20:14:58 DEBUG }, 20:14:58 DEBUG "value": [ 20:14:58 DEBUG 20:14:58 DEBUG 20:14:58 DEBUG 20:14:58 DEBUG [...] 20:14:58 DEBUG 20:14:58 DEBUG 20:14:58 DEBUG 20:14:58 DEBUG "instance": "10.142.0.43:9100" 20:14:58 DEBUG }, 20:14:58 DEBUG "value": [ 20:14:58 DEBUG 1713640498, 20:14:58 DEBUG "0.27546666666664615" 20:14:58 DEBUG ] 20:14:58 DEBUG } 20:14:58 DEBUG ] 20:14:58 DEBUG } 20:14:58 DEBUG } 20:14:58 INFO 20:14:58 INFO [loop_until]: kubectl --namespace=xlou top node 20:14:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:14:59 INFO [loop_until]: OK (rc = 0) 20:14:59 DEBUG --- stdout --- 20:14:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6218Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8126m 27% 7639Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17751m 59% 11387Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3343Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19890m 66% 14145Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5192Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18507m 61% 11960Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15725m 52% 2964Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20834m 69% 10906Mi 9% gke-xlou-cdm-ds-0f601751-j488 8736m 29% 7225Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18658m 62% 15819Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8603m 28% 7864Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20229m 67% 13716Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9445m 59% 6388Mi 10% 20:14:59 DEBUG --- stderr --- 20:14:59 DEBUG 20:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640499" 20:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640499" 20:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640499" 20:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640499" 20:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640499" 20:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640499" 20:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640499" 20:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640499" 20:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640499" 20:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640499" 20:14:59 INFO [http_cmd]: http status code OK 20:14:59 INFO [http_cmd]: http status code OK 20:14:59 INFO [http_cmd]: http status code OK 20:14:59 INFO [http_cmd]: http status code OK 20:14:59 DEBUG --- status code --- 20:14:59 DEBUG --- status code --- 20:14:59 INFO [http_cmd]: http status code OK 20:14:59 INFO [http_cmd]: http status code OK 20:14:59 INFO [http_cmd]: http status code OK 20:14:59 INFO [http_cmd]: http status code OK 20:14:59 DEBUG --- status code --- 20:14:59 INFO [http_cmd]: http status code OK 20:14:59 DEBUG --- status code --- 20:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:59 DEBUG --- status code --- 20:14:59 DEBUG --- status code --- 20:14:59 DEBUG --- status code --- 20:14:59 DEBUG --- status code --- 20:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:59 DEBUG --- status code --- 20:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:59 DEBUG --- http response --- 20:14:59 DEBUG --- http response --- 20:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:59 DEBUG --- http response --- 20:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:59 DEBUG --- http response --- 20:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:59 DEBUG --- http response --- 20:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:14:59 DEBUG --- http response --- 20:14:59 DEBUG --- http response --- 20:14:59 DEBUG --- http response --- 20:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:14:59 DEBUG --- http response --- 20:14:59 INFO [http_cmd]: http status code OK 20:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:59 DEBUG ----- output ----- 20:14:59 DEBUG ----- output ----- 20:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:59 DEBUG ----- output ----- 20:14:59 DEBUG --- status code --- 20:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:14:59 DEBUG ----- output ----- 20:14:59 DEBUG { 20:14:59 DEBUG { 20:14:59 DEBUG ----- output ----- 20:14:59 DEBUG ----- output ----- 20:14:59 DEBUG ----- output ----- 20:14:59 DEBUG ----- output ----- 20:14:59 DEBUG { 20:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:14:59 DEBUG ----- output ----- 20:14:59 DEBUG { 20:14:59 DEBUG "status": "success", 20:14:59 DEBUG "status": "success", 20:14:59 DEBUG { 20:14:59 DEBUG { 20:14:59 DEBUG { 20:14:59 DEBUG { 20:14:59 DEBUG "status": "success", 20:14:59 DEBUG --- http response --- 20:14:59 DEBUG { 20:14:59 DEBUG "status": "success", 20:14:59 DEBUG "data": { 20:14:59 DEBUG "data": { 20:14:59 DEBUG "status": "success", 20:14:59 DEBUG "data": { 20:14:59 DEBUG "status": "success", 20:14:59 DEBUG "status": "success", 20:14:59 DEBUG "data": { 20:14:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:14:59 DEBUG ----- output ----- 20:14:59 DEBUG "data": { 20:14:59 DEBUG "resultType": "vector", 20:14:59 DEBUG "resultType": "vector", 20:14:59 DEBUG "status": "success", 20:14:59 DEBUG "resultType": "vector", 20:14:59 DEBUG "data": { 20:14:59 DEBUG "data": { 20:14:59 DEBUG "resultType": "vector", 20:14:59 DEBUG { 20:14:59 DEBUG "resultType": "vector", 20:14:59 DEBUG "status": "success", 20:14:59 DEBUG "result": [ 20:14:59 DEBUG "result": [ 20:14:59 DEBUG { 20:14:59 DEBUG "result": [ 20:14:59 DEBUG "resultType": "vector", 20:14:59 DEBUG "resultType": "vector", 20:14:59 DEBUG "result": [ 20:14:59 DEBUG "status": "success", 20:14:59 DEBUG "data": { 20:14:59 DEBUG "data": { 20:14:59 DEBUG { 20:14:59 DEBUG "metric": { 20:14:59 DEBUG "pod": "ds-cts-0" 20:14:59 DEBUG { 20:14:59 DEBUG "result": [ 20:14:59 DEBUG "result": [ 20:14:59 DEBUG { 20:14:59 DEBUG "metric": { 20:14:59 DEBUG "result": [ 20:14:59 DEBUG "resultType": "vector", 20:14:59 DEBUG "metric": { 20:14:59 DEBUG }, 20:14:59 DEBUG "metric": { 20:14:59 DEBUG { 20:14:59 DEBUG "data": { 20:14:59 DEBUG { 20:14:59 DEBUG "resultType": "vector", 20:14:59 DEBUG "pod": "ds-idrepo-1" 20:14:59 DEBUG { 20:14:59 DEBUG "result": [ 20:14:59 DEBUG "instance": "10.142.0.107:9100" 20:14:59 DEBUG "value": [ 20:14:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:59 DEBUG "metric": { 20:14:59 DEBUG "resultType": "vector", 20:14:59 DEBUG "metric": { 20:14:59 DEBUG "result": [ 20:14:59 DEBUG }, 20:14:59 DEBUG "metric": { 20:14:59 DEBUG { 20:14:59 DEBUG }, 20:14:59 DEBUG 20:14:59 DEBUG }, 20:14:59 DEBUG "pod": "ds-idrepo-1" 20:14:59 DEBUG "result": [ 20:14:59 DEBUG "instance": "10.142.0.107:9100" 20:14:59 DEBUG { 20:14:59 DEBUG "value": [ 20:14:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:14:59 DEBUG "metric": { 20:14:59 DEBUG "value": [ 20:14:59 DEBUG 20:14:59 DEBUG "value": [ 20:14:59 DEBUG }, 20:14:59 DEBUG { 20:14:59 DEBUG }, 20:14:59 DEBUG "metric": { 20:14:59 DEBUG 20:14:59 DEBUG }, 20:14:59 DEBUG "value": [ 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG "value": [ 20:14:59 DEBUG "metric": { 20:14:59 DEBUG "value": [ 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG [...] 20:14:59 DEBUG 20:14:59 DEBUG "instance": "10.142.0.107:9100" 20:14:59 DEBUG 20:14:59 DEBUG "pod": "ds-cts-0" 20:14:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:14:59 DEBUG }, 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG }, 20:14:59 DEBUG 20:14:59 DEBUG }, 20:14:59 DEBUG 20:14:59 DEBUG "value": [ 20:14:59 DEBUG [...] 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG [...] 20:14:59 DEBUG "value": [ 20:14:59 DEBUG 20:14:59 DEBUG "value": [ 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG [...] 20:14:59 DEBUG "instance": "10.142.0.43:9100" 20:14:59 DEBUG [...] 20:14:59 DEBUG 20:14:59 DEBUG [...] 20:14:59 DEBUG 20:14:59 DEBUG [...] 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG }, 20:14:59 DEBUG 20:14:59 DEBUG }, 20:14:59 DEBUG 20:14:59 DEBUG [...] 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG [...] 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG "value": [ 20:14:59 DEBUG "pod": "ds-cts-2" 20:14:59 DEBUG "value": [ 20:14:59 DEBUG 1713640499, 20:14:59 DEBUG 20:14:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 1713640499, 20:14:59 DEBUG }, 20:14:59 DEBUG 20:14:59 DEBUG [...] 20:14:59 DEBUG 20:14:59 DEBUG }, 20:14:59 DEBUG "instance": "10.142.0.43:9100" 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG "0" 20:14:59 DEBUG "value": [ 20:14:59 DEBUG 20:14:59 DEBUG "85899796.96666667" 20:14:59 DEBUG 20:14:59 DEBUG "value": [ 20:14:59 DEBUG }, 20:14:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:14:59 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:14:59 DEBUG ] 20:14:59 DEBUG 1713640499, 20:14:59 DEBUG 20:14:59 DEBUG 20:14:59 DEBUG ] 20:14:59 DEBUG } 20:14:59 DEBUG "value": [ 20:14:59 DEBUG }, 20:14:59 DEBUG }, 20:14:59 DEBUG }, 20:14:59 DEBUG "value": [ 20:14:59 DEBUG "0.016933333333333335" 20:14:59 DEBUG "instance": "10.142.0.43:9100" 20:14:59 DEBUG "pod": "ds-cts-2" 20:14:59 DEBUG ] 20:14:59 DEBUG 1713640499, 20:14:59 DEBUG 1713640499, 20:14:59 DEBUG "value": [ 20:14:59 DEBUG "value": [ 20:14:59 DEBUG 1713640499, 20:14:59 DEBUG ] 20:14:59 DEBUG }, 20:14:59 DEBUG } 20:14:59 DEBUG }, 20:14:59 DEBUG } 20:14:59 DEBUG "85904125.93333334" 20:14:59 DEBUG "963391488" 20:14:59 DEBUG 1713640499, 20:14:59 DEBUG 1713640499, 20:14:59 DEBUG "49104.50830874655" 20:14:59 DEBUG } 20:14:59 DEBUG "value": [ 20:14:59 DEBUG ] 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG ] 20:14:59 DEBUG ] 20:14:59 DEBUG "0" 20:14:59 DEBUG "0.157" 20:14:59 DEBUG ] 20:14:59 DEBUG ] 20:14:59 DEBUG 1713640499, 20:14:59 DEBUG "value": [ 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG ] 20:14:59 DEBUG ] 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG "0.09933333333334528" 20:14:59 DEBUG 1713640499, 20:14:59 DEBUG ] 20:14:59 DEBUG ] 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG ] 20:14:59 DEBUG } 20:14:59 DEBUG ] 20:14:59 DEBUG "31.133333333333333" 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG ] 20:14:59 DEBUG ] 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG ] 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG ] 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG ] 20:14:59 DEBUG } 20:14:59 DEBUG } 20:14:59 DEBUG } 20:15:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640554" 20:15:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640554" 20:15:54 ERROR [http_cmd]: ERROR 20:15:54 ERROR --- status code --- 20:15:54 INFO [http_cmd]: http status code OK 20:15:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:15:54 DEBUG --- status code --- 20:15:54 ERROR --- http response --- 20:15:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:54 DEBUG --- http response --- 20:15:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:15:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:15:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640555" 20:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640555" 20:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640555" 20:15:55 INFO [http_cmd]: http status code OK 20:15:55 DEBUG --- status code --- 20:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:55 DEBUG --- http response --- 20:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:55 DEBUG ----- output ----- 20:15:55 DEBUG { 20:15:55 DEBUG "status": "success", 20:15:55 DEBUG "data": { 20:15:55 DEBUG "resultType": "vector", 20:15:55 DEBUG "result": [ 20:15:55 DEBUG { 20:15:55 DEBUG "metric": { 20:15:55 DEBUG "pod": "ds-idrepo-0" 20:15:55 DEBUG }, 20:15:55 DEBUG "value": [ 20:15:55 DEBUG 20:15:55 DEBUG 20:15:55 DEBUG 20:15:55 DEBUG [...] 20:15:55 DEBUG 20:15:55 DEBUG 20:15:55 DEBUG 20:15:55 DEBUG "pod": "ds-idrepo-2" 20:15:55 DEBUG }, 20:15:55 DEBUG "value": [ 20:15:55 DEBUG 1713640555, 20:15:55 DEBUG "0" 20:15:55 DEBUG ] 20:15:55 DEBUG } 20:15:55 DEBUG ] 20:15:55 DEBUG } 20:15:55 DEBUG } 20:15:55 INFO [http_cmd]: http status code OK 20:15:55 DEBUG --- status code --- 20:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:55 DEBUG --- http response --- 20:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:55 DEBUG ----- output ----- 20:15:55 DEBUG { 20:15:55 DEBUG "status": "success", 20:15:55 DEBUG "data": { 20:15:55 DEBUG "resultType": "vector", 20:15:55 DEBUG "result": [ 20:15:55 DEBUG { 20:15:55 DEBUG "metric": { 20:15:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:55 DEBUG }, 20:15:55 DEBUG "value": [ 20:15:55 DEBUG 20:15:55 DEBUG 20:15:55 DEBUG 20:15:55 DEBUG [...] 20:15:55 DEBUG 20:15:55 DEBUG 20:15:55 DEBUG 20:15:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:55 DEBUG }, 20:15:55 DEBUG "value": [ 20:15:55 DEBUG 1713640555, 20:15:55 DEBUG "2856.5666666666666" 20:15:55 DEBUG ] 20:15:55 DEBUG } 20:15:55 DEBUG ] 20:15:55 DEBUG } 20:15:55 DEBUG } 20:15:55 INFO [http_cmd]: http status code OK 20:15:55 DEBUG --- status code --- 20:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:55 DEBUG --- http response --- 20:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:55 DEBUG ----- output ----- 20:15:55 DEBUG { 20:15:55 DEBUG "status": "success", 20:15:55 DEBUG "data": { 20:15:55 DEBUG "resultType": "vector", 20:15:55 DEBUG "result": [ 20:15:55 DEBUG { 20:15:55 DEBUG "metric": { 20:15:55 DEBUG "pod": "ds-idrepo-0" 20:15:55 DEBUG }, 20:15:55 DEBUG "value": [ 20:15:55 DEBUG 20:15:55 DEBUG 20:15:55 DEBUG 20:15:55 DEBUG [...] 20:15:55 DEBUG 20:15:55 DEBUG 20:15:55 DEBUG 20:15:55 DEBUG "pod": "ds-idrepo-2" 20:15:55 DEBUG }, 20:15:55 DEBUG "value": [ 20:15:55 DEBUG 1713640555, 20:15:55 DEBUG "0" 20:15:55 DEBUG ] 20:15:55 DEBUG } 20:15:55 DEBUG ] 20:15:55 DEBUG } 20:15:55 DEBUG } 20:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640556" 20:15:56 INFO [http_cmd]: http status code OK 20:15:56 DEBUG --- status code --- 20:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:56 DEBUG --- http response --- 20:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:15:56 DEBUG ----- output ----- 20:15:56 DEBUG { 20:15:56 DEBUG "status": "success", 20:15:56 DEBUG "data": { 20:15:56 DEBUG "resultType": "vector", 20:15:56 DEBUG "result": [ 20:15:56 DEBUG { 20:15:56 DEBUG "metric": { 20:15:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:15:56 DEBUG }, 20:15:56 DEBUG "value": [ 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG [...] 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:15:56 DEBUG }, 20:15:56 DEBUG "value": [ 20:15:56 DEBUG 1713640556, 20:15:56 DEBUG "0" 20:15:56 DEBUG ] 20:15:56 DEBUG } 20:15:56 DEBUG ] 20:15:56 DEBUG } 20:15:56 DEBUG } 20:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640556" 20:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640556" 20:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640556" 20:15:56 INFO [http_cmd]: http status code OK 20:15:56 DEBUG --- status code --- 20:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:56 DEBUG --- http response --- 20:15:56 INFO [http_cmd]: http status code OK 20:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:56 DEBUG --- status code --- 20:15:56 DEBUG ----- output ----- 20:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:56 DEBUG { 20:15:56 DEBUG --- http response --- 20:15:56 DEBUG "status": "success", 20:15:56 DEBUG "data": { 20:15:56 DEBUG "resultType": "vector", 20:15:56 DEBUG "result": [ 20:15:56 DEBUG { 20:15:56 DEBUG "metric": { 20:15:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:56 DEBUG }, 20:15:56 DEBUG "value": [ 20:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:56 DEBUG 20:15:56 DEBUG ----- output ----- 20:15:56 DEBUG 20:15:56 DEBUG { 20:15:56 DEBUG 20:15:56 DEBUG "status": "success", 20:15:56 DEBUG [...] 20:15:56 DEBUG "data": { 20:15:56 DEBUG 20:15:56 DEBUG "resultType": "vector", 20:15:56 DEBUG 20:15:56 DEBUG "result": [ 20:15:56 DEBUG 20:15:56 DEBUG { 20:15:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:56 DEBUG "metric": { 20:15:56 DEBUG }, 20:15:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:15:56 DEBUG "value": [ 20:15:56 DEBUG }, 20:15:56 DEBUG 1713640556, 20:15:56 DEBUG "value": [ 20:15:56 DEBUG "2841.8" 20:15:56 DEBUG 20:15:56 DEBUG ] 20:15:56 DEBUG 20:15:56 DEBUG } 20:15:56 DEBUG 20:15:56 DEBUG ] 20:15:56 DEBUG [...] 20:15:56 DEBUG } 20:15:56 DEBUG 20:15:56 DEBUG } 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:15:56 DEBUG }, 20:15:56 DEBUG "value": [ 20:15:56 DEBUG 1713640556, 20:15:56 DEBUG "9.573498600214325" 20:15:56 DEBUG ] 20:15:56 DEBUG } 20:15:56 DEBUG ] 20:15:56 DEBUG } 20:15:56 DEBUG } 20:15:56 INFO [http_cmd]: http status code OK 20:15:56 DEBUG --- status code --- 20:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:56 DEBUG --- http response --- 20:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:56 DEBUG ----- output ----- 20:15:56 DEBUG { 20:15:56 DEBUG "status": "success", 20:15:56 DEBUG "data": { 20:15:56 DEBUG "resultType": "vector", 20:15:56 DEBUG "result": [ 20:15:56 DEBUG { 20:15:56 DEBUG "metric": { 20:15:56 DEBUG "instance": "10.142.0.107:9100" 20:15:56 DEBUG }, 20:15:56 DEBUG "value": [ 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG [...] 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG "instance": "10.142.0.43:9100" 20:15:56 DEBUG }, 20:15:56 DEBUG "value": [ 20:15:56 DEBUG 1713640556, 20:15:56 DEBUG "3712697760768" 20:15:56 DEBUG ] 20:15:56 DEBUG } 20:15:56 DEBUG ] 20:15:56 DEBUG } 20:15:56 DEBUG } 20:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640556" 20:15:56 INFO [http_cmd]: http status code OK 20:15:56 DEBUG --- status code --- 20:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:56 DEBUG --- http response --- 20:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:56 DEBUG ----- output ----- 20:15:56 DEBUG { 20:15:56 DEBUG "status": "success", 20:15:56 DEBUG "data": { 20:15:56 DEBUG "resultType": "vector", 20:15:56 DEBUG "result": [ 20:15:56 DEBUG { 20:15:56 DEBUG "metric": { 20:15:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:56 DEBUG }, 20:15:56 DEBUG "value": [ 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG [...] 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:56 DEBUG }, 20:15:56 DEBUG "value": [ 20:15:56 DEBUG 1713640556, 20:15:56 DEBUG "2841" 20:15:56 DEBUG ] 20:15:56 DEBUG } 20:15:56 DEBUG ] 20:15:56 DEBUG } 20:15:56 DEBUG } 20:15:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640556" 20:15:56 INFO [http_cmd]: http status code OK 20:15:56 DEBUG --- status code --- 20:15:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:56 DEBUG --- http response --- 20:15:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:56 DEBUG ----- output ----- 20:15:56 DEBUG { 20:15:56 DEBUG "status": "success", 20:15:56 DEBUG "data": { 20:15:56 DEBUG "resultType": "vector", 20:15:56 DEBUG "result": [ 20:15:56 DEBUG { 20:15:56 DEBUG "metric": { 20:15:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:56 DEBUG }, 20:15:56 DEBUG "value": [ 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG [...] 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG 20:15:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:56 DEBUG }, 20:15:56 DEBUG "value": [ 20:15:56 DEBUG 1713640556, 20:15:56 DEBUG "2840.2" 20:15:56 DEBUG ] 20:15:56 DEBUG } 20:15:56 DEBUG ] 20:15:56 DEBUG } 20:15:56 DEBUG } 20:15:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640557" 20:15:57 INFO [http_cmd]: http status code OK 20:15:57 DEBUG --- status code --- 20:15:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:57 DEBUG --- http response --- 20:15:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:15:57 DEBUG ----- output ----- 20:15:57 DEBUG { 20:15:57 DEBUG "status": "success", 20:15:57 DEBUG "data": { 20:15:57 DEBUG "resultType": "vector", 20:15:57 DEBUG "result": [ 20:15:57 DEBUG { 20:15:57 DEBUG "metric": { 20:15:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:15:57 DEBUG }, 20:15:57 DEBUG "value": [ 20:15:57 DEBUG 20:15:57 DEBUG 20:15:57 DEBUG 20:15:57 DEBUG [...] 20:15:57 DEBUG 20:15:57 DEBUG 20:15:57 DEBUG 20:15:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:15:57 DEBUG }, 20:15:57 DEBUG "value": [ 20:15:57 DEBUG 1713640557, 20:15:57 DEBUG "0.00002968694357215445" 20:15:57 DEBUG ] 20:15:57 DEBUG } 20:15:57 DEBUG ] 20:15:57 DEBUG } 20:15:57 DEBUG } 20:15:57 INFO 20:15:57 INFO [loop_until]: kubectl --namespace=xlou top pods 20:15:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:15:57 INFO [loop_until]: OK (rc = 0) 20:15:57 DEBUG --- stdout --- 20:15:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18571m 8665Mi am-767f6b67b5-qcqz2 18455m 9071Mi am-767f6b67b5-vbltz 20077m 8670Mi ds-cts-0 19958m 10961Mi ds-cts-1 18802m 10785Mi ds-cts-2 20505m 11050Mi ds-idrepo-0 7764m 3893Mi ds-idrepo-1 8043m 3882Mi ds-idrepo-2 8158m 3828Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9579m 831Mi 20:15:57 DEBUG --- stderr --- 20:15:57 DEBUG 20:15:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640557" 20:15:57 INFO [http_cmd]: http status code OK 20:15:57 DEBUG --- status code --- 20:15:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:57 DEBUG --- http response --- 20:15:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:57 DEBUG ----- output ----- 20:15:57 DEBUG { 20:15:57 DEBUG "status": "success", 20:15:57 DEBUG "data": { 20:15:57 DEBUG "resultType": "vector", 20:15:57 DEBUG "result": [ 20:15:57 DEBUG { 20:15:57 DEBUG "metric": { 20:15:57 DEBUG "pod": "ds-idrepo-0" 20:15:57 DEBUG }, 20:15:57 DEBUG "value": [ 20:15:57 DEBUG 20:15:57 DEBUG 20:15:57 DEBUG 20:15:57 DEBUG [...] 20:15:57 DEBUG 20:15:57 DEBUG 20:15:57 DEBUG 20:15:57 DEBUG "pod": "ds-idrepo-2" 20:15:57 DEBUG }, 20:15:57 DEBUG "value": [ 20:15:57 DEBUG 1713640557, 20:15:57 DEBUG "0" 20:15:57 DEBUG ] 20:15:57 DEBUG } 20:15:57 DEBUG ] 20:15:57 DEBUG } 20:15:57 DEBUG } 20:15:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640557" 20:15:57 INFO [http_cmd]: http status code OK 20:15:57 DEBUG --- status code --- 20:15:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:57 DEBUG --- http response --- 20:15:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:15:57 DEBUG ----- output ----- 20:15:57 DEBUG { 20:15:57 DEBUG "status": "success", 20:15:57 DEBUG "data": { 20:15:57 DEBUG "resultType": "vector", 20:15:57 DEBUG "result": [ 20:15:57 DEBUG { 20:15:57 DEBUG "metric": { 20:15:57 DEBUG "pod": "ds-cts-0" 20:15:57 DEBUG }, 20:15:57 DEBUG "value": [ 20:15:57 DEBUG 20:15:57 DEBUG 20:15:57 DEBUG 20:15:57 DEBUG [...] 20:15:57 DEBUG 20:15:57 DEBUG 20:15:57 DEBUG 20:15:57 DEBUG "pod": "ds-cts-2" 20:15:57 DEBUG }, 20:15:57 DEBUG "value": [ 20:15:57 DEBUG 1713640557, 20:15:57 DEBUG "0" 20:15:57 DEBUG ] 20:15:57 DEBUG } 20:15:57 DEBUG ] 20:15:57 DEBUG } 20:15:57 DEBUG } 20:15:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640558" 20:15:58 INFO [http_cmd]: http status code OK 20:15:58 DEBUG --- status code --- 20:15:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:58 DEBUG --- http response --- 20:15:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:15:58 DEBUG ----- output ----- 20:15:58 DEBUG { 20:15:58 DEBUG "status": "success", 20:15:58 DEBUG "data": { 20:15:58 DEBUG "resultType": "vector", 20:15:58 DEBUG "result": [ 20:15:58 DEBUG { 20:15:58 DEBUG "metric": { 20:15:58 DEBUG "pod": "ds-idrepo-0" 20:15:58 DEBUG }, 20:15:58 DEBUG "value": [ 20:15:58 DEBUG 20:15:58 DEBUG 20:15:58 DEBUG 20:15:58 DEBUG [...] 20:15:58 DEBUG 20:15:58 DEBUG 20:15:58 DEBUG 20:15:58 DEBUG "pod": "ds-cts-2" 20:15:58 DEBUG }, 20:15:58 DEBUG "value": [ 20:15:58 DEBUG 1713640558, 20:15:58 DEBUG "0" 20:15:58 DEBUG ] 20:15:58 DEBUG } 20:15:58 DEBUG ] 20:15:58 DEBUG } 20:15:58 DEBUG } 20:15:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640558" 20:15:58 INFO [http_cmd]: http status code OK 20:15:58 DEBUG --- status code --- 20:15:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:58 DEBUG --- http response --- 20:15:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:58 DEBUG ----- output ----- 20:15:58 DEBUG { 20:15:58 DEBUG "status": "success", 20:15:58 DEBUG "data": { 20:15:58 DEBUG "resultType": "vector", 20:15:58 DEBUG "result": [ 20:15:58 DEBUG { 20:15:58 DEBUG "metric": { 20:15:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:58 DEBUG }, 20:15:58 DEBUG "value": [ 20:15:58 DEBUG 20:15:58 DEBUG 20:15:58 DEBUG 20:15:58 DEBUG [...] 20:15:58 DEBUG 20:15:58 DEBUG 20:15:58 DEBUG 20:15:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:58 DEBUG }, 20:15:58 DEBUG "value": [ 20:15:58 DEBUG 1713640558, 20:15:58 DEBUG "510446" 20:15:58 DEBUG ] 20:15:58 DEBUG } 20:15:58 DEBUG ] 20:15:58 DEBUG } 20:15:58 DEBUG } 20:15:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640558" 20:15:58 INFO [http_cmd]: http status code OK 20:15:58 DEBUG --- status code --- 20:15:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:58 DEBUG --- http response --- 20:15:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:58 DEBUG ----- output ----- 20:15:58 DEBUG { 20:15:58 DEBUG "status": "success", 20:15:58 DEBUG "data": { 20:15:58 DEBUG "resultType": "vector", 20:15:58 DEBUG "result": [ 20:15:58 DEBUG { 20:15:58 DEBUG "metric": { 20:15:58 DEBUG "instance": "10.142.0.107:9100" 20:15:58 DEBUG }, 20:15:58 DEBUG "value": [ 20:15:58 DEBUG 20:15:58 DEBUG 20:15:58 DEBUG 20:15:58 DEBUG [...] 20:15:58 DEBUG 20:15:58 DEBUG 20:15:58 DEBUG 20:15:58 DEBUG "instance": "10.142.0.43:9100" 20:15:58 DEBUG }, 20:15:58 DEBUG "value": [ 20:15:58 DEBUG 1713640558, 20:15:58 DEBUG "0.2268333333332483" 20:15:58 DEBUG ] 20:15:58 DEBUG } 20:15:58 DEBUG ] 20:15:58 DEBUG } 20:15:58 DEBUG } 20:15:59 INFO 20:15:59 INFO [loop_until]: kubectl --namespace=xlou top node 20:15:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:15:59 INFO [loop_until]: OK (rc = 0) 20:15:59 DEBUG --- stdout --- 20:15:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6203Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8166m 27% 7638Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18310m 61% 11379Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3349Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19683m 65% 14153Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 137m 0% 5224Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18598m 62% 11959Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15800m 52% 2959Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20834m 69% 10894Mi 9% gke-xlou-cdm-ds-0f601751-j488 8759m 29% 7235Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18216m 61% 15824Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8659m 29% 7861Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19404m 64% 13709Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9890m 62% 6392Mi 10% 20:15:59 DEBUG --- stderr --- 20:15:59 DEBUG 20:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640559" 20:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640559" 20:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640559" 20:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640559" 20:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640559" 20:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640559" 20:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640559" 20:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640559" 20:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640559" 20:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640559" 20:15:59 INFO [http_cmd]: http status code OK 20:15:59 INFO [http_cmd]: http status code OK 20:15:59 DEBUG --- status code --- 20:15:59 DEBUG --- status code --- 20:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:59 INFO [http_cmd]: http status code OK 20:15:59 INFO [http_cmd]: http status code OK 20:15:59 INFO [http_cmd]: http status code OK 20:15:59 INFO [http_cmd]: http status code OK 20:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:59 DEBUG --- http response --- 20:15:59 DEBUG --- status code --- 20:15:59 DEBUG --- status code --- 20:15:59 DEBUG --- status code --- 20:15:59 DEBUG --- status code --- 20:15:59 DEBUG --- http response --- 20:15:59 INFO [http_cmd]: http status code OK 20:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:59 DEBUG --- http response --- 20:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:15:59 DEBUG --- status code --- 20:15:59 DEBUG --- http response --- 20:15:59 DEBUG ----- output ----- 20:15:59 DEBUG --- http response --- 20:15:59 INFO [http_cmd]: http status code OK 20:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:59 DEBUG --- http response --- 20:15:59 DEBUG ----- output ----- 20:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:59 DEBUG { 20:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:15:59 INFO [http_cmd]: http status code OK 20:15:59 DEBUG --- status code --- 20:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:15:59 DEBUG ----- output ----- 20:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:59 DEBUG { 20:15:59 DEBUG --- http response --- 20:15:59 DEBUG "status": "success", 20:15:59 DEBUG ----- output ----- 20:15:59 DEBUG --- status code --- 20:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:59 INFO [http_cmd]: http status code OK 20:15:59 DEBUG ----- output ----- 20:15:59 DEBUG { 20:15:59 DEBUG ----- output ----- 20:15:59 DEBUG "status": "success", 20:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:15:59 DEBUG "data": { 20:15:59 DEBUG { 20:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:59 DEBUG --- http response --- 20:15:59 DEBUG --- status code --- 20:15:59 DEBUG { 20:15:59 DEBUG "status": "success", 20:15:59 DEBUG { 20:15:59 DEBUG "data": { 20:15:59 DEBUG ----- output ----- 20:15:59 DEBUG "resultType": "vector", 20:15:59 DEBUG "status": "success", 20:15:59 DEBUG --- http response --- 20:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:15:59 DEBUG "status": "success", 20:15:59 DEBUG "data": { 20:15:59 DEBUG "status": "success", 20:15:59 DEBUG "resultType": "vector", 20:15:59 DEBUG { 20:15:59 DEBUG "result": [ 20:15:59 DEBUG "data": { 20:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:15:59 DEBUG ----- output ----- 20:15:59 DEBUG --- http response --- 20:15:59 DEBUG "data": { 20:15:59 DEBUG "resultType": "vector", 20:15:59 DEBUG "data": { 20:15:59 DEBUG "result": [ 20:15:59 DEBUG "status": "success", 20:15:59 DEBUG { 20:15:59 DEBUG "resultType": "vector", 20:15:59 DEBUG ----- output ----- 20:15:59 DEBUG { 20:15:59 DEBUG "resultType": "vector", 20:15:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:15:59 DEBUG "result": [ 20:15:59 DEBUG "resultType": "vector", 20:15:59 DEBUG { 20:15:59 DEBUG "data": { 20:15:59 DEBUG "metric": { 20:15:59 DEBUG "result": [ 20:15:59 DEBUG { 20:15:59 DEBUG "status": "success", 20:15:59 DEBUG "result": [ 20:15:59 DEBUG ----- output ----- 20:15:59 DEBUG { 20:15:59 DEBUG "result": [ 20:15:59 DEBUG "metric": { 20:15:59 DEBUG "resultType": "vector", 20:15:59 DEBUG "instance": "10.142.0.107:9100" 20:15:59 DEBUG { 20:15:59 DEBUG "status": "success", 20:15:59 DEBUG "data": { 20:15:59 DEBUG { 20:15:59 DEBUG { 20:15:59 DEBUG "metric": { 20:15:59 DEBUG { 20:15:59 DEBUG "pod": "ds-cts-0" 20:15:59 DEBUG "result": [ 20:15:59 DEBUG }, 20:15:59 DEBUG "metric": { 20:15:59 DEBUG "data": { 20:15:59 DEBUG "resultType": "vector", 20:15:59 DEBUG "metric": { 20:15:59 DEBUG "status": "success", 20:15:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:59 DEBUG }, 20:15:59 DEBUG }, 20:15:59 DEBUG { 20:15:59 DEBUG "value": [ 20:15:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:15:59 DEBUG "resultType": "vector", 20:15:59 DEBUG "result": [ 20:15:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:15:59 DEBUG "data": { 20:15:59 DEBUG "metric": { 20:15:59 DEBUG "value": [ 20:15:59 DEBUG "value": [ 20:15:59 DEBUG "metric": { 20:15:59 DEBUG 20:15:59 DEBUG }, 20:15:59 DEBUG "result": [ 20:15:59 DEBUG { 20:15:59 DEBUG }, 20:15:59 DEBUG "resultType": "vector", 20:15:59 DEBUG "result": [ 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG { 20:15:59 DEBUG "metric": { 20:15:59 DEBUG "value": [ 20:15:59 DEBUG { 20:15:59 DEBUG "instance": "10.142.0.107:9100" 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG }, 20:15:59 DEBUG "value": [ 20:15:59 DEBUG [...] 20:15:59 DEBUG "metric": { 20:15:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:15:59 DEBUG 20:15:59 DEBUG "metric": { 20:15:59 DEBUG "pod": "ds-cts-0" 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG "value": [ 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG "instance": "10.142.0.107:9100" 20:15:59 DEBUG }, 20:15:59 DEBUG 20:15:59 DEBUG }, 20:15:59 DEBUG }, 20:15:59 DEBUG [...] 20:15:59 DEBUG [...] 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG }, 20:15:59 DEBUG "value": [ 20:15:59 DEBUG 20:15:59 DEBUG "value": [ 20:15:59 DEBUG "value": [ 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG "value": [ 20:15:59 DEBUG 20:15:59 DEBUG [...] 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG [...] 20:15:59 DEBUG "instance": "10.142.0.43:9100" 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG [...] 20:15:59 DEBUG 20:15:59 DEBUG }, 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:59 DEBUG "pod": "ds-cts-2" 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG "value": [ 20:15:59 DEBUG 20:15:59 DEBUG [...] 20:15:59 DEBUG 20:15:59 DEBUG [...] 20:15:59 DEBUG [...] 20:15:59 DEBUG }, 20:15:59 DEBUG }, 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 1713640559, 20:15:59 DEBUG [...] 20:15:59 DEBUG 20:15:59 DEBUG "pod": "ds-cts-0" 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG "value": [ 20:15:59 DEBUG "value": [ 20:15:59 DEBUG 20:15:59 DEBUG "pod": "ds-cts-0" 20:15:59 DEBUG "85867402.8" 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG }, 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG 1713640559, 20:15:59 DEBUG 1713640559, 20:15:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:15:59 DEBUG }, 20:15:59 DEBUG ] 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG "value": [ 20:15:59 DEBUG 20:15:59 DEBUG 20:15:59 DEBUG "0" 20:15:59 DEBUG "0.027633333333333333" 20:15:59 DEBUG }, 20:15:59 DEBUG "value": [ 20:15:59 DEBUG } 20:15:59 DEBUG 20:15:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:15:59 DEBUG 1713640559, 20:15:59 DEBUG "instance": "10.142.0.43:9100" 20:15:59 DEBUG "pod": "ds-cts-2" 20:15:59 DEBUG ] 20:15:59 DEBUG ] 20:15:59 DEBUG "value": [ 20:15:59 DEBUG 1713640559, 20:15:59 DEBUG ] 20:15:59 DEBUG "instance": "10.142.0.43:9100" 20:15:59 DEBUG }, 20:15:59 DEBUG "287868594.4655212" 20:15:59 DEBUG }, 20:15:59 DEBUG }, 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG 1713640559, 20:15:59 DEBUG "37898388.54148964" 20:15:59 DEBUG } 20:15:59 DEBUG }, 20:15:59 DEBUG "value": [ 20:15:59 DEBUG ] 20:15:59 DEBUG "value": [ 20:15:59 DEBUG "value": [ 20:15:59 DEBUG ] 20:15:59 DEBUG ] 20:15:59 DEBUG "0.157" 20:15:59 DEBUG ] 20:15:59 DEBUG } 20:15:59 DEBUG "value": [ 20:15:59 DEBUG 1713640559, 20:15:59 DEBUG } 20:15:59 DEBUG 1713640559, 20:15:59 DEBUG 1713640559, 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG 1713640559, 20:15:59 DEBUG "964603904" 20:15:59 DEBUG ] 20:15:59 DEBUG "85869176.96666667" 20:15:59 DEBUG "34.5" 20:15:59 DEBUG } 20:15:59 DEBUG ] 20:15:59 DEBUG ] 20:15:59 DEBUG "0.07199999999999136" 20:15:59 DEBUG ] 20:15:59 DEBUG } 20:15:59 DEBUG ] 20:15:59 DEBUG ] 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG ] 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG ] 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG ] 20:15:59 DEBUG ] 20:15:59 DEBUG ] 20:15:59 DEBUG } 20:15:59 DEBUG ] 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG } 20:15:59 DEBUG } 20:16:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640614" 20:16:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640614" 20:16:54 INFO [http_cmd]: http status code OK 20:16:54 ERROR [http_cmd]: ERROR 20:16:54 DEBUG --- status code --- 20:16:54 ERROR --- status code --- 20:16:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:16:54 DEBUG --- http response --- 20:16:54 ERROR --- http response --- 20:16:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:16:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:16:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640615" 20:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640615" 20:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640615" 20:16:55 INFO [http_cmd]: http status code OK 20:16:55 DEBUG --- status code --- 20:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:55 DEBUG --- http response --- 20:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:55 DEBUG ----- output ----- 20:16:55 DEBUG { 20:16:55 DEBUG "status": "success", 20:16:55 DEBUG "data": { 20:16:55 DEBUG "resultType": "vector", 20:16:55 DEBUG "result": [ 20:16:55 DEBUG { 20:16:55 DEBUG "metric": { 20:16:55 DEBUG "pod": "ds-idrepo-0" 20:16:55 DEBUG }, 20:16:55 DEBUG "value": [ 20:16:55 DEBUG 20:16:55 DEBUG 20:16:55 DEBUG 20:16:55 DEBUG [...] 20:16:55 DEBUG 20:16:55 DEBUG 20:16:55 DEBUG 20:16:55 DEBUG "pod": "ds-idrepo-2" 20:16:55 DEBUG }, 20:16:55 DEBUG "value": [ 20:16:55 DEBUG 1713640615, 20:16:55 DEBUG "0" 20:16:55 DEBUG ] 20:16:55 DEBUG } 20:16:55 DEBUG ] 20:16:55 DEBUG } 20:16:55 DEBUG } 20:16:55 INFO [http_cmd]: http status code OK 20:16:55 INFO [http_cmd]: http status code OK 20:16:55 DEBUG --- status code --- 20:16:55 DEBUG --- status code --- 20:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:55 DEBUG --- http response --- 20:16:55 DEBUG --- http response --- 20:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:55 DEBUG ----- output ----- 20:16:55 DEBUG ----- output ----- 20:16:55 DEBUG { 20:16:55 DEBUG { 20:16:55 DEBUG "status": "success", 20:16:55 DEBUG "status": "success", 20:16:55 DEBUG "data": { 20:16:55 DEBUG "data": { 20:16:55 DEBUG "resultType": "vector", 20:16:55 DEBUG "resultType": "vector", 20:16:55 DEBUG "result": [ 20:16:55 DEBUG "result": [ 20:16:55 DEBUG { 20:16:55 DEBUG { 20:16:55 DEBUG "metric": { 20:16:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:55 DEBUG "metric": { 20:16:55 DEBUG }, 20:16:55 DEBUG "value": [ 20:16:55 DEBUG "pod": "ds-idrepo-0" 20:16:55 DEBUG 20:16:55 DEBUG }, 20:16:55 DEBUG 20:16:55 DEBUG "value": [ 20:16:55 DEBUG 20:16:55 DEBUG 20:16:55 DEBUG [...] 20:16:55 DEBUG 20:16:55 DEBUG 20:16:55 DEBUG 20:16:55 DEBUG 20:16:55 DEBUG [...] 20:16:55 DEBUG 20:16:55 DEBUG 20:16:55 DEBUG 20:16:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:55 DEBUG 20:16:55 DEBUG }, 20:16:55 DEBUG "pod": "ds-idrepo-2" 20:16:55 DEBUG "value": [ 20:16:55 DEBUG }, 20:16:55 DEBUG 1713640615, 20:16:55 DEBUG "value": [ 20:16:55 DEBUG "2828.366666666667" 20:16:55 DEBUG 1713640615, 20:16:55 DEBUG ] 20:16:55 DEBUG "0" 20:16:55 DEBUG } 20:16:55 DEBUG ] 20:16:55 DEBUG ] 20:16:55 DEBUG } 20:16:55 DEBUG } 20:16:55 DEBUG ] 20:16:55 DEBUG } 20:16:55 DEBUG } 20:16:55 DEBUG } 20:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640616" 20:16:56 INFO [http_cmd]: http status code OK 20:16:56 DEBUG --- status code --- 20:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:56 DEBUG --- http response --- 20:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:56 DEBUG ----- output ----- 20:16:56 DEBUG { 20:16:56 DEBUG "status": "success", 20:16:56 DEBUG "data": { 20:16:56 DEBUG "resultType": "vector", 20:16:56 DEBUG "result": [ 20:16:56 DEBUG { 20:16:56 DEBUG "metric": { 20:16:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:16:56 DEBUG }, 20:16:56 DEBUG "value": [ 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG [...] 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:16:56 DEBUG }, 20:16:56 DEBUG "value": [ 20:16:56 DEBUG 1713640616, 20:16:56 DEBUG "0" 20:16:56 DEBUG ] 20:16:56 DEBUG } 20:16:56 DEBUG ] 20:16:56 DEBUG } 20:16:56 DEBUG } 20:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640616" 20:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640616" 20:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640616" 20:16:56 INFO [http_cmd]: http status code OK 20:16:56 DEBUG --- status code --- 20:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:56 DEBUG --- http response --- 20:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:56 INFO [http_cmd]: http status code OK 20:16:56 DEBUG ----- output ----- 20:16:56 DEBUG --- status code --- 20:16:56 DEBUG { 20:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:56 DEBUG --- http response --- 20:16:56 DEBUG "status": "success", 20:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:56 INFO [http_cmd]: http status code OK 20:16:56 DEBUG "data": { 20:16:56 DEBUG ----- output ----- 20:16:56 DEBUG --- status code --- 20:16:56 DEBUG "resultType": "vector", 20:16:56 DEBUG { 20:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:56 DEBUG "result": [ 20:16:56 DEBUG "status": "success", 20:16:56 DEBUG --- http response --- 20:16:56 DEBUG { 20:16:56 DEBUG "data": { 20:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:56 DEBUG "metric": { 20:16:56 DEBUG "resultType": "vector", 20:16:56 DEBUG ----- output ----- 20:16:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:56 DEBUG "result": [ 20:16:56 DEBUG { 20:16:56 DEBUG }, 20:16:56 DEBUG { 20:16:56 DEBUG "status": "success", 20:16:56 DEBUG "value": [ 20:16:56 DEBUG "metric": { 20:16:56 DEBUG "data": { 20:16:56 DEBUG 20:16:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:16:56 DEBUG "resultType": "vector", 20:16:56 DEBUG 20:16:56 DEBUG }, 20:16:56 DEBUG "result": [ 20:16:56 DEBUG 20:16:56 DEBUG "value": [ 20:16:56 DEBUG { 20:16:56 DEBUG [...] 20:16:56 DEBUG 20:16:56 DEBUG "metric": { 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG "instance": "10.142.0.107:9100" 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG }, 20:16:56 DEBUG 20:16:56 DEBUG [...] 20:16:56 DEBUG "value": [ 20:16:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG }, 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG "value": [ 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG 1713640616, 20:16:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:16:56 DEBUG [...] 20:16:56 DEBUG "2831.8" 20:16:56 DEBUG }, 20:16:56 DEBUG 20:16:56 DEBUG ] 20:16:56 DEBUG "value": [ 20:16:56 DEBUG 20:16:56 DEBUG } 20:16:56 DEBUG 1713640616, 20:16:56 DEBUG 20:16:56 DEBUG ] 20:16:56 DEBUG "9.50802015419597" 20:16:56 DEBUG "instance": "10.142.0.43:9100" 20:16:56 DEBUG } 20:16:56 DEBUG ] 20:16:56 DEBUG }, 20:16:56 DEBUG } 20:16:56 DEBUG } 20:16:56 DEBUG "value": [ 20:16:56 DEBUG ] 20:16:56 DEBUG 1713640616, 20:16:56 DEBUG } 20:16:56 DEBUG "3716432671744" 20:16:56 DEBUG } 20:16:56 DEBUG ] 20:16:56 DEBUG } 20:16:56 DEBUG ] 20:16:56 DEBUG } 20:16:56 DEBUG } 20:16:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640616" 20:16:56 INFO [http_cmd]: http status code OK 20:16:56 DEBUG --- status code --- 20:16:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:56 DEBUG --- http response --- 20:16:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:56 DEBUG ----- output ----- 20:16:56 DEBUG { 20:16:56 DEBUG "status": "success", 20:16:56 DEBUG "data": { 20:16:56 DEBUG "resultType": "vector", 20:16:56 DEBUG "result": [ 20:16:56 DEBUG { 20:16:56 DEBUG "metric": { 20:16:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:56 DEBUG }, 20:16:56 DEBUG "value": [ 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG [...] 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG 20:16:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:56 DEBUG }, 20:16:56 DEBUG "value": [ 20:16:56 DEBUG 1713640616, 20:16:56 DEBUG "2832.5333333333333" 20:16:56 DEBUG ] 20:16:56 DEBUG } 20:16:56 DEBUG ] 20:16:56 DEBUG } 20:16:56 DEBUG } 20:16:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640617" 20:16:57 INFO [http_cmd]: http status code OK 20:16:57 DEBUG --- status code --- 20:16:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:57 DEBUG --- http response --- 20:16:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:57 DEBUG ----- output ----- 20:16:57 DEBUG { 20:16:57 DEBUG "status": "success", 20:16:57 DEBUG "data": { 20:16:57 DEBUG "resultType": "vector", 20:16:57 DEBUG "result": [ 20:16:57 DEBUG { 20:16:57 DEBUG "metric": { 20:16:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:57 DEBUG }, 20:16:57 DEBUG "value": [ 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG [...] 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:57 DEBUG }, 20:16:57 DEBUG "value": [ 20:16:57 DEBUG 1713640617, 20:16:57 DEBUG "2833.2" 20:16:57 DEBUG ] 20:16:57 DEBUG } 20:16:57 DEBUG ] 20:16:57 DEBUG } 20:16:57 DEBUG } 20:16:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640617" 20:16:57 INFO [http_cmd]: http status code OK 20:16:57 DEBUG --- status code --- 20:16:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:57 DEBUG --- http response --- 20:16:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:57 DEBUG ----- output ----- 20:16:57 DEBUG { 20:16:57 DEBUG "status": "success", 20:16:57 DEBUG "data": { 20:16:57 DEBUG "resultType": "vector", 20:16:57 DEBUG "result": [ 20:16:57 DEBUG { 20:16:57 DEBUG "metric": { 20:16:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:16:57 DEBUG }, 20:16:57 DEBUG "value": [ 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG [...] 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:16:57 DEBUG }, 20:16:57 DEBUG "value": [ 20:16:57 DEBUG 1713640617, 20:16:57 DEBUG "0.005720735737846578" 20:16:57 DEBUG ] 20:16:57 DEBUG } 20:16:57 DEBUG ] 20:16:57 DEBUG } 20:16:57 DEBUG } 20:16:57 INFO 20:16:57 INFO [loop_until]: kubectl --namespace=xlou top pods 20:16:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:16:57 INFO [loop_until]: OK (rc = 0) 20:16:57 DEBUG --- stdout --- 20:16:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18210m 8665Mi am-767f6b67b5-qcqz2 18484m 9070Mi am-767f6b67b5-vbltz 19923m 8671Mi ds-cts-0 20305m 10964Mi ds-cts-1 18113m 10796Mi ds-cts-2 20759m 11045Mi ds-idrepo-0 7828m 3893Mi ds-idrepo-1 7978m 3882Mi ds-idrepo-2 7747m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9511m 832Mi 20:16:57 DEBUG --- stderr --- 20:16:57 DEBUG 20:16:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640617" 20:16:57 INFO [http_cmd]: http status code OK 20:16:57 DEBUG --- status code --- 20:16:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:57 DEBUG --- http response --- 20:16:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:57 DEBUG ----- output ----- 20:16:57 DEBUG { 20:16:57 DEBUG "status": "success", 20:16:57 DEBUG "data": { 20:16:57 DEBUG "resultType": "vector", 20:16:57 DEBUG "result": [ 20:16:57 DEBUG { 20:16:57 DEBUG "metric": { 20:16:57 DEBUG "pod": "ds-idrepo-0" 20:16:57 DEBUG }, 20:16:57 DEBUG "value": [ 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG [...] 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG "pod": "ds-idrepo-2" 20:16:57 DEBUG }, 20:16:57 DEBUG "value": [ 20:16:57 DEBUG 1713640617, 20:16:57 DEBUG "0" 20:16:57 DEBUG ] 20:16:57 DEBUG } 20:16:57 DEBUG ] 20:16:57 DEBUG } 20:16:57 DEBUG } 20:16:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640617" 20:16:57 INFO [http_cmd]: http status code OK 20:16:57 DEBUG --- status code --- 20:16:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:57 DEBUG --- http response --- 20:16:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:16:57 DEBUG ----- output ----- 20:16:57 DEBUG { 20:16:57 DEBUG "status": "success", 20:16:57 DEBUG "data": { 20:16:57 DEBUG "resultType": "vector", 20:16:57 DEBUG "result": [ 20:16:57 DEBUG { 20:16:57 DEBUG "metric": { 20:16:57 DEBUG "pod": "ds-cts-0" 20:16:57 DEBUG }, 20:16:57 DEBUG "value": [ 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG [...] 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG 20:16:57 DEBUG "pod": "ds-cts-2" 20:16:57 DEBUG }, 20:16:57 DEBUG "value": [ 20:16:57 DEBUG 1713640617, 20:16:57 DEBUG "0" 20:16:57 DEBUG ] 20:16:57 DEBUG } 20:16:57 DEBUG ] 20:16:57 DEBUG } 20:16:57 DEBUG } 20:16:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640618" 20:16:58 INFO [http_cmd]: http status code OK 20:16:58 DEBUG --- status code --- 20:16:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:58 DEBUG --- http response --- 20:16:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:16:58 DEBUG ----- output ----- 20:16:58 DEBUG { 20:16:58 DEBUG "status": "success", 20:16:58 DEBUG "data": { 20:16:58 DEBUG "resultType": "vector", 20:16:58 DEBUG "result": [ 20:16:58 DEBUG { 20:16:58 DEBUG "metric": { 20:16:58 DEBUG "pod": "ds-idrepo-0" 20:16:58 DEBUG }, 20:16:58 DEBUG "value": [ 20:16:58 DEBUG 20:16:58 DEBUG 20:16:58 DEBUG 20:16:58 DEBUG [...] 20:16:58 DEBUG 20:16:58 DEBUG 20:16:58 DEBUG 20:16:58 DEBUG "pod": "ds-cts-2" 20:16:58 DEBUG }, 20:16:58 DEBUG "value": [ 20:16:58 DEBUG 1713640618, 20:16:58 DEBUG "0" 20:16:58 DEBUG ] 20:16:58 DEBUG } 20:16:58 DEBUG ] 20:16:58 DEBUG } 20:16:58 DEBUG } 20:16:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640618" 20:16:58 INFO [http_cmd]: http status code OK 20:16:58 DEBUG --- status code --- 20:16:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:58 DEBUG --- http response --- 20:16:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:58 DEBUG ----- output ----- 20:16:58 DEBUG { 20:16:58 DEBUG "status": "success", 20:16:58 DEBUG "data": { 20:16:58 DEBUG "resultType": "vector", 20:16:58 DEBUG "result": [ 20:16:58 DEBUG { 20:16:58 DEBUG "metric": { 20:16:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:58 DEBUG }, 20:16:58 DEBUG "value": [ 20:16:58 DEBUG 20:16:58 DEBUG 20:16:58 DEBUG 20:16:58 DEBUG [...] 20:16:58 DEBUG 20:16:58 DEBUG 20:16:58 DEBUG 20:16:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:58 DEBUG }, 20:16:58 DEBUG "value": [ 20:16:58 DEBUG 1713640618, 20:16:58 DEBUG "510501" 20:16:58 DEBUG ] 20:16:58 DEBUG } 20:16:58 DEBUG ] 20:16:58 DEBUG } 20:16:58 DEBUG } 20:16:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640618" 20:16:58 INFO [http_cmd]: http status code OK 20:16:58 DEBUG --- status code --- 20:16:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:58 DEBUG --- http response --- 20:16:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:58 DEBUG ----- output ----- 20:16:58 DEBUG { 20:16:58 DEBUG "status": "success", 20:16:58 DEBUG "data": { 20:16:58 DEBUG "resultType": "vector", 20:16:58 DEBUG "result": [ 20:16:58 DEBUG { 20:16:58 DEBUG "metric": { 20:16:58 DEBUG "instance": "10.142.0.107:9100" 20:16:58 DEBUG }, 20:16:58 DEBUG "value": [ 20:16:58 DEBUG 20:16:58 DEBUG 20:16:58 DEBUG 20:16:58 DEBUG [...] 20:16:58 DEBUG 20:16:58 DEBUG 20:16:58 DEBUG 20:16:58 DEBUG "instance": "10.142.0.43:9100" 20:16:58 DEBUG }, 20:16:58 DEBUG "value": [ 20:16:58 DEBUG 1713640618, 20:16:58 DEBUG "0.16303333333334724" 20:16:58 DEBUG ] 20:16:58 DEBUG } 20:16:58 DEBUG ] 20:16:58 DEBUG } 20:16:58 DEBUG } 20:16:59 INFO 20:16:59 INFO [loop_until]: kubectl --namespace=xlou top node 20:16:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:16:59 INFO [loop_until]: OK (rc = 0) 20:16:59 DEBUG --- stdout --- 20:16:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6211Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8210m 27% 7645Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18305m 61% 11382Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3327Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20045m 67% 14158Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5210Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18529m 62% 11959Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15662m 52% 2978Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20976m 70% 10892Mi 9% gke-xlou-cdm-ds-0f601751-j488 8737m 29% 7228Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18047m 60% 15835Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8411m 28% 7853Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20482m 68% 13698Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9759m 61% 6397Mi 10% 20:16:59 DEBUG --- stderr --- 20:16:59 DEBUG 20:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640619" 20:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640619" 20:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640619" 20:16:59 INFO [http_cmd]: http status code OK 20:16:59 DEBUG --- status code --- 20:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:59 DEBUG --- http response --- 20:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:59 DEBUG ----- output ----- 20:16:59 DEBUG { 20:16:59 DEBUG "status": "success", 20:16:59 DEBUG "data": { 20:16:59 DEBUG "resultType": "vector", 20:16:59 DEBUG "result": [ 20:16:59 DEBUG { 20:16:59 DEBUG "metric": { 20:16:59 DEBUG "instance": "10.142.0.107:9100" 20:16:59 DEBUG }, 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG [...] 20:16:59 DEBUG 20:16:59 INFO [http_cmd]: http status code OK 20:16:59 DEBUG 20:16:59 DEBUG --- status code --- 20:16:59 DEBUG 20:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:59 DEBUG "instance": "10.142.0.43:9100" 20:16:59 DEBUG --- http response --- 20:16:59 DEBUG }, 20:16:59 DEBUG "value": [ 20:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:59 DEBUG 1713640619, 20:16:59 DEBUG ----- output ----- 20:16:59 DEBUG "85187553.86666667" 20:16:59 DEBUG { 20:16:59 DEBUG ] 20:16:59 DEBUG "status": "success", 20:16:59 DEBUG } 20:16:59 DEBUG "data": { 20:16:59 DEBUG ] 20:16:59 DEBUG "resultType": "vector", 20:16:59 DEBUG } 20:16:59 DEBUG "result": [ 20:16:59 DEBUG } 20:16:59 DEBUG { 20:16:59 DEBUG "metric": { 20:16:59 DEBUG "instance": "10.142.0.107:9100" 20:16:59 DEBUG }, 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 20:16:59 INFO [http_cmd]: http status code OK 20:16:59 DEBUG 20:16:59 DEBUG --- status code --- 20:16:59 DEBUG 20:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:59 DEBUG [...] 20:16:59 DEBUG --- http response --- 20:16:59 DEBUG 20:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:59 DEBUG 20:16:59 DEBUG ----- output ----- 20:16:59 DEBUG 20:16:59 DEBUG { 20:16:59 DEBUG "instance": "10.142.0.43:9100" 20:16:59 DEBUG "status": "success", 20:16:59 DEBUG }, 20:16:59 DEBUG "data": { 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 1713640619, 20:16:59 DEBUG "85190166.46666667" 20:16:59 DEBUG "resultType": "vector", 20:16:59 DEBUG "result": [ 20:16:59 DEBUG ] 20:16:59 DEBUG { 20:16:59 DEBUG } 20:16:59 DEBUG "metric": { 20:16:59 DEBUG ] 20:16:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:16:59 DEBUG } 20:16:59 DEBUG }, 20:16:59 DEBUG } 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG [...] 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:16:59 DEBUG }, 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 1713640619, 20:16:59 DEBUG "966574080" 20:16:59 DEBUG ] 20:16:59 DEBUG } 20:16:59 DEBUG ] 20:16:59 DEBUG } 20:16:59 DEBUG } 20:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640619" 20:16:59 INFO [http_cmd]: http status code OK 20:16:59 DEBUG --- status code --- 20:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:59 DEBUG --- http response --- 20:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:16:59 DEBUG ----- output ----- 20:16:59 DEBUG { 20:16:59 DEBUG "status": "success", 20:16:59 DEBUG "data": { 20:16:59 DEBUG "resultType": "vector", 20:16:59 DEBUG "result": [ 20:16:59 DEBUG { 20:16:59 DEBUG "metric": { 20:16:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:16:59 DEBUG }, 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG [...] 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:16:59 DEBUG }, 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 1713640619, 20:16:59 DEBUG "0" 20:16:59 DEBUG ] 20:16:59 DEBUG } 20:16:59 DEBUG ] 20:16:59 DEBUG } 20:16:59 DEBUG } 20:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640619" 20:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640619" 20:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640619" 20:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640619" 20:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640619" 20:16:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640619" 20:16:59 INFO [http_cmd]: http status code OK 20:16:59 DEBUG --- status code --- 20:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:59 DEBUG --- http response --- 20:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:59 DEBUG ----- output ----- 20:16:59 DEBUG { 20:16:59 DEBUG "status": "success", 20:16:59 DEBUG "data": { 20:16:59 DEBUG "resultType": "vector", 20:16:59 DEBUG "result": [ 20:16:59 DEBUG { 20:16:59 DEBUG "metric": { 20:16:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:59 DEBUG }, 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG [...] 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:59 DEBUG }, 20:16:59 DEBUG "value": [ 20:16:59 INFO [http_cmd]: http status code OK 20:16:59 DEBUG 1713640619, 20:16:59 INFO [http_cmd]: http status code OK 20:16:59 DEBUG --- status code --- 20:16:59 DEBUG "0" 20:16:59 DEBUG --- status code --- 20:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:59 DEBUG ] 20:16:59 DEBUG } 20:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:59 INFO [http_cmd]: http status code OK 20:16:59 DEBUG --- http response --- 20:16:59 DEBUG ] 20:16:59 INFO [http_cmd]: http status code OK 20:16:59 DEBUG --- http response --- 20:16:59 DEBUG --- status code --- 20:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:16:59 DEBUG } 20:16:59 INFO [http_cmd]: http status code OK 20:16:59 DEBUG --- status code --- 20:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:59 DEBUG ----- output ----- 20:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:16:59 DEBUG } 20:16:59 DEBUG --- status code --- 20:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:59 DEBUG --- http response --- 20:16:59 DEBUG { 20:16:59 DEBUG ----- output ----- 20:16:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:16:59 DEBUG --- http response --- 20:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:16:59 DEBUG "status": "success", 20:16:59 DEBUG "data": { 20:16:59 DEBUG --- http response --- 20:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:16:59 DEBUG ----- output ----- 20:16:59 DEBUG { 20:16:59 DEBUG "resultType": "vector", 20:16:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:16:59 DEBUG ----- output ----- 20:16:59 DEBUG { 20:16:59 DEBUG "status": "success", 20:16:59 DEBUG "result": [ 20:16:59 DEBUG ----- output ----- 20:16:59 DEBUG { 20:16:59 DEBUG "status": "success", 20:16:59 DEBUG "data": { 20:16:59 DEBUG { 20:16:59 DEBUG { 20:16:59 DEBUG "status": "success", 20:16:59 DEBUG "data": { 20:16:59 DEBUG "resultType": "vector", 20:16:59 DEBUG "metric": { 20:16:59 DEBUG "status": "success", 20:16:59 DEBUG "data": { 20:16:59 DEBUG "resultType": "vector", 20:16:59 DEBUG "result": [ 20:16:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:16:59 DEBUG "data": { 20:16:59 DEBUG "resultType": "vector", 20:16:59 DEBUG "result": [ 20:16:59 DEBUG { 20:16:59 DEBUG }, 20:16:59 DEBUG "resultType": "vector", 20:16:59 DEBUG "result": [ 20:16:59 DEBUG { 20:16:59 DEBUG "metric": { 20:16:59 DEBUG "value": [ 20:16:59 DEBUG "result": [ 20:16:59 DEBUG { 20:16:59 DEBUG "metric": { 20:16:59 DEBUG "pod": "ds-cts-0" 20:16:59 DEBUG 20:16:59 DEBUG { 20:16:59 DEBUG "metric": { 20:16:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:16:59 DEBUG }, 20:16:59 DEBUG 20:16:59 DEBUG "metric": { 20:16:59 DEBUG "pod": "ds-cts-0" 20:16:59 DEBUG }, 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 20:16:59 DEBUG "instance": "10.142.0.107:9100" 20:16:59 DEBUG }, 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 20:16:59 DEBUG [...] 20:16:59 DEBUG }, 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG [...] 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG [...] 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG [...] 20:16:59 DEBUG 20:16:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:16:59 DEBUG "pod": "ds-cts-2" 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG }, 20:16:59 DEBUG }, 20:16:59 DEBUG [...] 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG "value": [ 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 20:16:59 DEBUG 20:16:59 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:16:59 DEBUG 1713640619, 20:16:59 DEBUG 1713640619, 20:16:59 DEBUG 20:16:59 DEBUG "pod": "ds-cts-2" 20:16:59 DEBUG }, 20:16:59 DEBUG "0.157" 20:16:59 DEBUG "0.11989999999999999" 20:16:59 DEBUG 20:16:59 DEBUG }, 20:16:59 DEBUG "value": [ 20:16:59 DEBUG ] 20:16:59 DEBUG ] 20:16:59 DEBUG "instance": "10.142.0.43:9100" 20:16:59 DEBUG "value": [ 20:16:59 DEBUG 1713640619, 20:16:59 DEBUG } 20:16:59 DEBUG } 20:16:59 DEBUG }, 20:16:59 DEBUG 1713640619, 20:16:59 DEBUG "0" 20:16:59 DEBUG ] 20:16:59 DEBUG } 20:16:59 DEBUG "value": [ 20:16:59 DEBUG "29.5" 20:16:59 DEBUG ] 20:16:59 DEBUG } 20:16:59 DEBUG ] 20:16:59 DEBUG 1713640619, 20:16:59 DEBUG ] 20:16:59 DEBUG } 20:16:59 DEBUG } 20:16:59 DEBUG "0.06366666666666182" 20:16:59 DEBUG } 20:16:59 DEBUG ] 20:16:59 DEBUG } 20:16:59 DEBUG ] 20:16:59 DEBUG ] 20:16:59 DEBUG } 20:16:59 DEBUG } 20:16:59 DEBUG } 20:16:59 DEBUG } 20:16:59 DEBUG ] 20:16:59 DEBUG } 20:16:59 DEBUG } 20:16:59 DEBUG } 20:17:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640674" 20:17:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640674" 20:17:54 ERROR [http_cmd]: ERROR 20:17:54 ERROR --- status code --- 20:17:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:17:54 ERROR --- http response --- 20:17:54 INFO [http_cmd]: http status code OK 20:17:54 DEBUG --- status code --- 20:17:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:17:54 DEBUG --- http response --- 20:17:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:17:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640675" 20:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640675" 20:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640675" 20:17:55 INFO [http_cmd]: http status code OK 20:17:55 DEBUG --- status code --- 20:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:55 DEBUG --- http response --- 20:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:55 DEBUG ----- output ----- 20:17:55 DEBUG { 20:17:55 DEBUG "status": "success", 20:17:55 DEBUG "data": { 20:17:55 DEBUG "resultType": "vector", 20:17:55 DEBUG "result": [ 20:17:55 DEBUG { 20:17:55 DEBUG "metric": { 20:17:55 DEBUG "pod": "ds-idrepo-0" 20:17:55 DEBUG }, 20:17:55 DEBUG "value": [ 20:17:55 DEBUG 20:17:55 DEBUG 20:17:55 DEBUG 20:17:55 DEBUG [...] 20:17:55 DEBUG 20:17:55 DEBUG 20:17:55 DEBUG 20:17:55 DEBUG "pod": "ds-idrepo-2" 20:17:55 DEBUG }, 20:17:55 DEBUG "value": [ 20:17:55 DEBUG 1713640675, 20:17:55 DEBUG "0" 20:17:55 DEBUG ] 20:17:55 DEBUG } 20:17:55 DEBUG ] 20:17:55 DEBUG } 20:17:55 DEBUG } 20:17:55 INFO [http_cmd]: http status code OK 20:17:55 DEBUG --- status code --- 20:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:55 DEBUG --- http response --- 20:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:55 DEBUG ----- output ----- 20:17:55 DEBUG { 20:17:55 DEBUG "status": "success", 20:17:55 DEBUG "data": { 20:17:55 DEBUG "resultType": "vector", 20:17:55 DEBUG "result": [ 20:17:55 DEBUG { 20:17:55 DEBUG "metric": { 20:17:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:55 DEBUG }, 20:17:55 DEBUG "value": [ 20:17:55 DEBUG 20:17:55 DEBUG 20:17:55 DEBUG 20:17:55 DEBUG [...] 20:17:55 DEBUG 20:17:55 DEBUG 20:17:55 DEBUG 20:17:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:55 DEBUG }, 20:17:55 DEBUG "value": [ 20:17:55 DEBUG 1713640675, 20:17:55 DEBUG "2833.9" 20:17:55 DEBUG ] 20:17:55 DEBUG } 20:17:55 DEBUG ] 20:17:55 DEBUG } 20:17:55 DEBUG } 20:17:55 INFO [http_cmd]: http status code OK 20:17:55 DEBUG --- status code --- 20:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:55 DEBUG --- http response --- 20:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:55 DEBUG ----- output ----- 20:17:55 DEBUG { 20:17:55 DEBUG "status": "success", 20:17:55 DEBUG "data": { 20:17:55 DEBUG "resultType": "vector", 20:17:55 DEBUG "result": [ 20:17:55 DEBUG { 20:17:55 DEBUG "metric": { 20:17:55 DEBUG "pod": "ds-idrepo-0" 20:17:55 DEBUG }, 20:17:55 DEBUG "value": [ 20:17:55 DEBUG 20:17:55 DEBUG 20:17:55 DEBUG 20:17:55 DEBUG [...] 20:17:55 DEBUG 20:17:55 DEBUG 20:17:55 DEBUG 20:17:55 DEBUG "pod": "ds-idrepo-2" 20:17:55 DEBUG }, 20:17:55 DEBUG "value": [ 20:17:55 DEBUG 1713640675, 20:17:55 DEBUG "0" 20:17:55 DEBUG ] 20:17:55 DEBUG } 20:17:55 DEBUG ] 20:17:55 DEBUG } 20:17:55 DEBUG } 20:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640676" 20:17:56 INFO [http_cmd]: http status code OK 20:17:56 DEBUG --- status code --- 20:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:56 DEBUG --- http response --- 20:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:17:56 DEBUG ----- output ----- 20:17:56 DEBUG { 20:17:56 DEBUG "status": "success", 20:17:56 DEBUG "data": { 20:17:56 DEBUG "resultType": "vector", 20:17:56 DEBUG "result": [ 20:17:56 DEBUG { 20:17:56 DEBUG "metric": { 20:17:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:17:56 DEBUG }, 20:17:56 DEBUG "value": [ 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG [...] 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:17:56 DEBUG }, 20:17:56 DEBUG "value": [ 20:17:56 DEBUG 1713640676, 20:17:56 DEBUG "0" 20:17:56 DEBUG ] 20:17:56 DEBUG } 20:17:56 DEBUG ] 20:17:56 DEBUG } 20:17:56 DEBUG } 20:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640676" 20:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640676" 20:17:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640676" 20:17:56 INFO [http_cmd]: http status code OK 20:17:56 DEBUG --- status code --- 20:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:56 DEBUG --- http response --- 20:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:56 DEBUG ----- output ----- 20:17:56 DEBUG { 20:17:56 DEBUG "status": "success", 20:17:56 DEBUG "data": { 20:17:56 DEBUG "resultType": "vector", 20:17:56 DEBUG "result": [ 20:17:56 DEBUG { 20:17:56 DEBUG "metric": { 20:17:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:56 DEBUG }, 20:17:56 DEBUG "value": [ 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG [...] 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:56 DEBUG }, 20:17:56 DEBUG "value": [ 20:17:56 DEBUG 1713640676, 20:17:56 DEBUG "2827.633333333333" 20:17:56 DEBUG ] 20:17:56 DEBUG } 20:17:56 DEBUG ] 20:17:56 DEBUG } 20:17:56 DEBUG } 20:17:56 INFO [http_cmd]: http status code OK 20:17:56 DEBUG --- status code --- 20:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:56 DEBUG --- http response --- 20:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:56 DEBUG ----- output ----- 20:17:56 DEBUG { 20:17:56 DEBUG "status": "success", 20:17:56 DEBUG "data": { 20:17:56 DEBUG "resultType": "vector", 20:17:56 DEBUG "result": [ 20:17:56 DEBUG { 20:17:56 DEBUG "metric": { 20:17:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:17:56 DEBUG }, 20:17:56 DEBUG "value": [ 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG [...] 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:17:56 DEBUG }, 20:17:56 DEBUG "value": [ 20:17:56 DEBUG 1713640676, 20:17:56 DEBUG "9.544450391286263" 20:17:56 DEBUG ] 20:17:56 DEBUG } 20:17:56 DEBUG ] 20:17:56 DEBUG } 20:17:56 DEBUG } 20:17:56 INFO [http_cmd]: http status code OK 20:17:56 DEBUG --- status code --- 20:17:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:56 DEBUG --- http response --- 20:17:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:56 DEBUG ----- output ----- 20:17:56 DEBUG { 20:17:56 DEBUG "status": "success", 20:17:56 DEBUG "data": { 20:17:56 DEBUG "resultType": "vector", 20:17:56 DEBUG "result": [ 20:17:56 DEBUG { 20:17:56 DEBUG "metric": { 20:17:56 DEBUG "instance": "10.142.0.107:9100" 20:17:56 DEBUG }, 20:17:56 DEBUG "value": [ 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG [...] 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG 20:17:56 DEBUG "instance": "10.142.0.43:9100" 20:17:56 DEBUG }, 20:17:56 DEBUG "value": [ 20:17:56 DEBUG 1713640676, 20:17:56 DEBUG "3720545821696" 20:17:56 DEBUG ] 20:17:56 DEBUG } 20:17:56 DEBUG ] 20:17:56 DEBUG } 20:17:56 DEBUG } 20:17:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640677" 20:17:57 INFO [http_cmd]: http status code OK 20:17:57 DEBUG --- status code --- 20:17:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:57 DEBUG --- http response --- 20:17:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:57 DEBUG ----- output ----- 20:17:57 DEBUG { 20:17:57 DEBUG "status": "success", 20:17:57 DEBUG "data": { 20:17:57 DEBUG "resultType": "vector", 20:17:57 DEBUG "result": [ 20:17:57 DEBUG { 20:17:57 DEBUG "metric": { 20:17:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:57 DEBUG }, 20:17:57 DEBUG "value": [ 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG [...] 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:57 DEBUG }, 20:17:57 DEBUG "value": [ 20:17:57 DEBUG 1713640677, 20:17:57 DEBUG "2827.8" 20:17:57 DEBUG ] 20:17:57 DEBUG } 20:17:57 DEBUG ] 20:17:57 DEBUG } 20:17:57 DEBUG } 20:17:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640677" 20:17:57 INFO [http_cmd]: http status code OK 20:17:57 DEBUG --- status code --- 20:17:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:57 DEBUG --- http response --- 20:17:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:57 DEBUG ----- output ----- 20:17:57 DEBUG { 20:17:57 DEBUG "status": "success", 20:17:57 DEBUG "data": { 20:17:57 DEBUG "resultType": "vector", 20:17:57 DEBUG "result": [ 20:17:57 DEBUG { 20:17:57 DEBUG "metric": { 20:17:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:57 DEBUG }, 20:17:57 DEBUG "value": [ 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG [...] 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:57 DEBUG }, 20:17:57 DEBUG "value": [ 20:17:57 DEBUG 1713640677, 20:17:57 DEBUG "2828.8333333333335" 20:17:57 DEBUG ] 20:17:57 DEBUG } 20:17:57 DEBUG ] 20:17:57 DEBUG } 20:17:57 DEBUG } 20:17:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640677" 20:17:57 INFO [http_cmd]: http status code OK 20:17:57 DEBUG --- status code --- 20:17:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:57 DEBUG --- http response --- 20:17:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:17:57 DEBUG ----- output ----- 20:17:57 DEBUG { 20:17:57 DEBUG "status": "success", 20:17:57 DEBUG "data": { 20:17:57 DEBUG "resultType": "vector", 20:17:57 DEBUG "result": [ 20:17:57 DEBUG { 20:17:57 DEBUG "metric": { 20:17:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:17:57 DEBUG }, 20:17:57 DEBUG "value": [ 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG [...] 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:17:57 DEBUG }, 20:17:57 DEBUG "value": [ 20:17:57 DEBUG 1713640677, 20:17:57 DEBUG "0.00003602554319451173" 20:17:57 DEBUG ] 20:17:57 DEBUG } 20:17:57 DEBUG ] 20:17:57 DEBUG } 20:17:57 DEBUG } 20:17:57 INFO 20:17:57 INFO [loop_until]: kubectl --namespace=xlou top pods 20:17:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:17:57 INFO [loop_until]: OK (rc = 0) 20:17:57 DEBUG --- stdout --- 20:17:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18123m 8663Mi am-767f6b67b5-qcqz2 18121m 9070Mi am-767f6b67b5-vbltz 19573m 8670Mi ds-cts-0 19406m 10966Mi ds-cts-1 17810m 10788Mi ds-cts-2 19290m 11051Mi ds-idrepo-0 7892m 3893Mi ds-idrepo-1 7680m 3882Mi ds-idrepo-2 7962m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9588m 834Mi 20:17:57 DEBUG --- stderr --- 20:17:57 DEBUG 20:17:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640677" 20:17:57 INFO [http_cmd]: http status code OK 20:17:57 DEBUG --- status code --- 20:17:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:57 DEBUG --- http response --- 20:17:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:57 DEBUG ----- output ----- 20:17:57 DEBUG { 20:17:57 DEBUG "status": "success", 20:17:57 DEBUG "data": { 20:17:57 DEBUG "resultType": "vector", 20:17:57 DEBUG "result": [ 20:17:57 DEBUG { 20:17:57 DEBUG "metric": { 20:17:57 DEBUG "pod": "ds-idrepo-0" 20:17:57 DEBUG }, 20:17:57 DEBUG "value": [ 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG [...] 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG 20:17:57 DEBUG "pod": "ds-idrepo-2" 20:17:57 DEBUG }, 20:17:57 DEBUG "value": [ 20:17:57 DEBUG 1713640677, 20:17:57 DEBUG "0" 20:17:57 DEBUG ] 20:17:57 DEBUG } 20:17:57 DEBUG ] 20:17:57 DEBUG } 20:17:57 DEBUG } 20:17:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640678" 20:17:58 INFO [http_cmd]: http status code OK 20:17:58 DEBUG --- status code --- 20:17:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:58 DEBUG --- http response --- 20:17:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:17:58 DEBUG ----- output ----- 20:17:58 DEBUG { 20:17:58 DEBUG "status": "success", 20:17:58 DEBUG "data": { 20:17:58 DEBUG "resultType": "vector", 20:17:58 DEBUG "result": [ 20:17:58 DEBUG { 20:17:58 DEBUG "metric": { 20:17:58 DEBUG "pod": "ds-cts-0" 20:17:58 DEBUG }, 20:17:58 DEBUG "value": [ 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG [...] 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG "pod": "ds-cts-2" 20:17:58 DEBUG }, 20:17:58 DEBUG "value": [ 20:17:58 DEBUG 1713640678, 20:17:58 DEBUG "0" 20:17:58 DEBUG ] 20:17:58 DEBUG } 20:17:58 DEBUG ] 20:17:58 DEBUG } 20:17:58 DEBUG } 20:17:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640678" 20:17:58 INFO [http_cmd]: http status code OK 20:17:58 DEBUG --- status code --- 20:17:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:58 DEBUG --- http response --- 20:17:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:17:58 DEBUG ----- output ----- 20:17:58 DEBUG { 20:17:58 DEBUG "status": "success", 20:17:58 DEBUG "data": { 20:17:58 DEBUG "resultType": "vector", 20:17:58 DEBUG "result": [ 20:17:58 DEBUG { 20:17:58 DEBUG "metric": { 20:17:58 DEBUG "pod": "ds-idrepo-0" 20:17:58 DEBUG }, 20:17:58 DEBUG "value": [ 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG [...] 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG "pod": "ds-cts-2" 20:17:58 DEBUG }, 20:17:58 DEBUG "value": [ 20:17:58 DEBUG 1713640678, 20:17:58 DEBUG "0" 20:17:58 DEBUG ] 20:17:58 DEBUG } 20:17:58 DEBUG ] 20:17:58 DEBUG } 20:17:58 DEBUG } 20:17:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640678" 20:17:58 INFO [http_cmd]: http status code OK 20:17:58 DEBUG --- status code --- 20:17:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:58 DEBUG --- http response --- 20:17:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:58 DEBUG ----- output ----- 20:17:58 DEBUG { 20:17:58 DEBUG "status": "success", 20:17:58 DEBUG "data": { 20:17:58 DEBUG "resultType": "vector", 20:17:58 DEBUG "result": [ 20:17:58 DEBUG { 20:17:58 DEBUG "metric": { 20:17:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:58 DEBUG }, 20:17:58 DEBUG "value": [ 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG [...] 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:58 DEBUG }, 20:17:58 DEBUG "value": [ 20:17:58 DEBUG 1713640678, 20:17:58 DEBUG "510120" 20:17:58 DEBUG ] 20:17:58 DEBUG } 20:17:58 DEBUG ] 20:17:58 DEBUG } 20:17:58 DEBUG } 20:17:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640678" 20:17:58 INFO [http_cmd]: http status code OK 20:17:58 DEBUG --- status code --- 20:17:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:58 DEBUG --- http response --- 20:17:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:58 DEBUG ----- output ----- 20:17:58 DEBUG { 20:17:58 DEBUG "status": "success", 20:17:58 DEBUG "data": { 20:17:58 DEBUG "resultType": "vector", 20:17:58 DEBUG "result": [ 20:17:58 DEBUG { 20:17:58 DEBUG "metric": { 20:17:58 DEBUG "instance": "10.142.0.107:9100" 20:17:58 DEBUG }, 20:17:58 DEBUG "value": [ 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG [...] 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG 20:17:58 DEBUG "instance": "10.142.0.43:9100" 20:17:58 DEBUG }, 20:17:58 DEBUG "value": [ 20:17:58 DEBUG 1713640678, 20:17:58 DEBUG "0.1559666666666686" 20:17:58 DEBUG ] 20:17:58 DEBUG } 20:17:58 DEBUG ] 20:17:58 DEBUG } 20:17:58 DEBUG } 20:17:59 INFO 20:17:59 INFO [loop_until]: kubectl --namespace=xlou top node 20:17:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:17:59 INFO [loop_until]: OK (rc = 0) 20:17:59 DEBUG --- stdout --- 20:17:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6210Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8198m 27% 7645Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18270m 61% 11371Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 77m 0% 3331Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19526m 65% 14147Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5196Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18491m 61% 11954Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15660m 52% 2976Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20749m 69% 10889Mi 9% gke-xlou-cdm-ds-0f601751-j488 8736m 29% 7221Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18342m 61% 15827Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8634m 28% 7864Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20095m 67% 13710Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9765m 61% 6393Mi 10% 20:17:59 DEBUG --- stderr --- 20:17:59 DEBUG 20:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640679" 20:17:59 INFO [http_cmd]: http status code OK 20:17:59 DEBUG --- status code --- 20:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:59 DEBUG --- http response --- 20:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:59 DEBUG ----- output ----- 20:17:59 DEBUG { 20:17:59 DEBUG "status": "success", 20:17:59 DEBUG "data": { 20:17:59 DEBUG "resultType": "vector", 20:17:59 DEBUG "result": [ 20:17:59 DEBUG { 20:17:59 DEBUG "metric": { 20:17:59 DEBUG "instance": "10.142.0.107:9100" 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG [...] 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG "instance": "10.142.0.43:9100" 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 1713640679, 20:17:59 DEBUG "85516946.9" 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG } 20:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640679" 20:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640679" 20:17:59 INFO [http_cmd]: http status code OK 20:17:59 DEBUG --- status code --- 20:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:59 DEBUG --- http response --- 20:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:59 DEBUG ----- output ----- 20:17:59 DEBUG { 20:17:59 DEBUG "status": "success", 20:17:59 DEBUG "data": { 20:17:59 DEBUG "resultType": "vector", 20:17:59 DEBUG "result": [ 20:17:59 DEBUG { 20:17:59 DEBUG "metric": { 20:17:59 DEBUG "instance": "10.142.0.107:9100" 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG [...] 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG "instance": "10.142.0.43:9100" 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 1713640679, 20:17:59 DEBUG "85519564.1" 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG } 20:17:59 INFO [http_cmd]: http status code OK 20:17:59 DEBUG --- status code --- 20:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:59 DEBUG --- http response --- 20:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:59 DEBUG ----- output ----- 20:17:59 DEBUG { 20:17:59 DEBUG "status": "success", 20:17:59 DEBUG "data": { 20:17:59 DEBUG "resultType": "vector", 20:17:59 DEBUG "result": [ 20:17:59 DEBUG { 20:17:59 DEBUG "metric": { 20:17:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG [...] 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 1713640679, 20:17:59 DEBUG "968941568" 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG } 20:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640679" 20:17:59 INFO [http_cmd]: http status code OK 20:17:59 DEBUG --- status code --- 20:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:59 DEBUG --- http response --- 20:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:17:59 DEBUG ----- output ----- 20:17:59 DEBUG { 20:17:59 DEBUG "status": "success", 20:17:59 DEBUG "data": { 20:17:59 DEBUG "resultType": "vector", 20:17:59 DEBUG "result": [ 20:17:59 DEBUG { 20:17:59 DEBUG "metric": { 20:17:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG [...] 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 1713640679, 20:17:59 DEBUG "0" 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG } 20:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640679" 20:17:59 INFO [http_cmd]: http status code OK 20:17:59 DEBUG --- status code --- 20:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:59 DEBUG --- http response --- 20:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:59 DEBUG ----- output ----- 20:17:59 DEBUG { 20:17:59 DEBUG "status": "success", 20:17:59 DEBUG "data": { 20:17:59 DEBUG "resultType": "vector", 20:17:59 DEBUG "result": [ 20:17:59 DEBUG { 20:17:59 DEBUG "metric": { 20:17:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG [...] 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 1713640679, 20:17:59 DEBUG "0" 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG } 20:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640679" 20:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640679" 20:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640679" 20:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640679" 20:17:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640679" 20:17:59 INFO [http_cmd]: http status code OK 20:17:59 INFO [http_cmd]: http status code OK 20:17:59 DEBUG --- status code --- 20:17:59 INFO [http_cmd]: http status code OK 20:17:59 DEBUG --- status code --- 20:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:59 DEBUG --- status code --- 20:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:59 DEBUG --- http response --- 20:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:59 DEBUG --- http response --- 20:17:59 DEBUG --- http response --- 20:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:17:59 DEBUG ----- output ----- 20:17:59 INFO [http_cmd]: http status code OK 20:17:59 DEBUG ----- output ----- 20:17:59 DEBUG { 20:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:59 DEBUG --- status code --- 20:17:59 DEBUG { 20:17:59 DEBUG "status": "success", 20:17:59 DEBUG ----- output ----- 20:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:59 DEBUG "status": "success", 20:17:59 DEBUG "data": { 20:17:59 DEBUG { 20:17:59 INFO [http_cmd]: http status code OK 20:17:59 DEBUG --- http response --- 20:17:59 DEBUG "data": { 20:17:59 DEBUG "resultType": "vector", 20:17:59 DEBUG "status": "success", 20:17:59 DEBUG --- status code --- 20:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:17:59 DEBUG "resultType": "vector", 20:17:59 DEBUG "result": [ 20:17:59 DEBUG "data": { 20:17:59 DEBUG "resultType": "vector", 20:17:59 DEBUG ----- output ----- 20:17:59 DEBUG "result": [ 20:17:59 DEBUG { 20:17:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:17:59 DEBUG "result": [ 20:17:59 DEBUG { 20:17:59 DEBUG { 20:17:59 DEBUG "metric": { 20:17:59 DEBUG --- http response --- 20:17:59 DEBUG { 20:17:59 DEBUG "status": "success", 20:17:59 DEBUG "metric": { 20:17:59 DEBUG "pod": "ds-cts-0" 20:17:59 DEBUG "metric": { 20:17:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:17:59 DEBUG "data": { 20:17:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG ----- output ----- 20:17:59 DEBUG "resultType": "vector", 20:17:59 DEBUG 20:17:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:17:59 DEBUG }, 20:17:59 DEBUG { 20:17:59 DEBUG "result": [ 20:17:59 DEBUG 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG "status": "success", 20:17:59 DEBUG { 20:17:59 DEBUG 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 20:17:59 DEBUG "data": { 20:17:59 DEBUG "metric": { 20:17:59 DEBUG [...] 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG "resultType": "vector", 20:17:59 DEBUG "pod": "ds-cts-0" 20:17:59 DEBUG }, 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG "result": [ 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG [...] 20:17:59 DEBUG { 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG [...] 20:17:59 DEBUG 20:17:59 DEBUG "metric": { 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG "instance": "10.142.0.107:9100" 20:17:59 DEBUG 20:17:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG }, 20:17:59 DEBUG [...] 20:17:59 DEBUG }, 20:17:59 DEBUG 20:17:59 DEBUG "pod": "ds-cts-2" 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 20:17:59 DEBUG "value": [ 20:17:59 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:17:59 DEBUG }, 20:17:59 DEBUG 20:17:59 DEBUG 20:17:59 DEBUG 1713640679, 20:17:59 DEBUG }, 20:17:59 DEBUG "value": [ 20:17:59 DEBUG 1713640679, 20:17:59 DEBUG 20:17:59 DEBUG "0.157" 20:17:59 DEBUG "value": [ 20:17:59 DEBUG "0.03831480888888889" 20:17:59 DEBUG 20:17:59 DEBUG "pod": "ds-cts-2" 20:17:59 DEBUG ] 20:17:59 DEBUG 1713640679, 20:17:59 DEBUG ] 20:17:59 DEBUG 20:17:59 DEBUG }, 20:17:59 DEBUG } 20:17:59 DEBUG "0" 20:17:59 DEBUG } 20:17:59 DEBUG [...] 20:17:59 DEBUG "value": [ 20:17:59 DEBUG ] 20:17:59 DEBUG ] 20:17:59 DEBUG ] 20:17:59 DEBUG 20:17:59 DEBUG 1713640679, 20:17:59 DEBUG } 20:17:59 DEBUG } 20:17:59 DEBUG } 20:17:59 DEBUG 20:17:59 DEBUG "33.93333333333333" 20:17:59 DEBUG } 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG "instance": "10.142.0.43:9100" 20:17:59 DEBUG } 20:17:59 DEBUG } 20:17:59 DEBUG }, 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG "value": [ 20:17:59 DEBUG } 20:17:59 DEBUG 1713640679, 20:17:59 DEBUG "0.06166666666665795" 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG ] 20:17:59 DEBUG } 20:17:59 DEBUG } 20:18:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640734" 20:18:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640734" 20:18:54 ERROR [http_cmd]: ERROR 20:18:54 ERROR --- status code --- 20:18:54 INFO [http_cmd]: http status code OK 20:18:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:18:54 DEBUG --- status code --- 20:18:54 ERROR --- http response --- 20:18:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:18:54 DEBUG --- http response --- 20:18:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:18:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640735" 20:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640735" 20:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640735" 20:18:55 INFO [http_cmd]: http status code OK 20:18:55 DEBUG --- status code --- 20:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:55 DEBUG --- http response --- 20:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:55 DEBUG ----- output ----- 20:18:55 DEBUG { 20:18:55 DEBUG "status": "success", 20:18:55 DEBUG "data": { 20:18:55 DEBUG "resultType": "vector", 20:18:55 DEBUG "result": [ 20:18:55 DEBUG { 20:18:55 DEBUG "metric": { 20:18:55 DEBUG "pod": "ds-idrepo-0" 20:18:55 DEBUG }, 20:18:55 DEBUG "value": [ 20:18:55 DEBUG 20:18:55 DEBUG 20:18:55 DEBUG 20:18:55 DEBUG [...] 20:18:55 DEBUG 20:18:55 DEBUG 20:18:55 DEBUG 20:18:55 DEBUG "pod": "ds-idrepo-2" 20:18:55 DEBUG }, 20:18:55 DEBUG "value": [ 20:18:55 DEBUG 1713640735, 20:18:55 DEBUG "0" 20:18:55 DEBUG ] 20:18:55 DEBUG } 20:18:55 DEBUG ] 20:18:55 DEBUG } 20:18:55 DEBUG } 20:18:55 INFO [http_cmd]: http status code OK 20:18:55 INFO [http_cmd]: http status code OK 20:18:55 DEBUG --- status code --- 20:18:55 DEBUG --- status code --- 20:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:55 DEBUG --- http response --- 20:18:55 DEBUG --- http response --- 20:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:55 DEBUG ----- output ----- 20:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:55 DEBUG { 20:18:55 DEBUG ----- output ----- 20:18:55 DEBUG "status": "success", 20:18:55 DEBUG { 20:18:55 DEBUG "data": { 20:18:55 DEBUG "status": "success", 20:18:55 DEBUG "resultType": "vector", 20:18:55 DEBUG "data": { 20:18:55 DEBUG "result": [ 20:18:55 DEBUG "resultType": "vector", 20:18:55 DEBUG { 20:18:55 DEBUG "result": [ 20:18:55 DEBUG "metric": { 20:18:55 DEBUG { 20:18:55 DEBUG "pod": "ds-idrepo-0" 20:18:55 DEBUG "metric": { 20:18:55 DEBUG }, 20:18:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:55 DEBUG "value": [ 20:18:55 DEBUG }, 20:18:55 DEBUG 20:18:55 DEBUG "value": [ 20:18:55 DEBUG 20:18:55 DEBUG 20:18:55 DEBUG 20:18:55 DEBUG 20:18:55 DEBUG [...] 20:18:55 DEBUG 20:18:55 DEBUG 20:18:55 DEBUG [...] 20:18:55 DEBUG 20:18:55 DEBUG 20:18:55 DEBUG 20:18:55 DEBUG 20:18:55 DEBUG "pod": "ds-idrepo-2" 20:18:55 DEBUG 20:18:55 DEBUG }, 20:18:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:55 DEBUG "value": [ 20:18:55 DEBUG }, 20:18:55 DEBUG 1713640735, 20:18:55 DEBUG "value": [ 20:18:55 DEBUG "0" 20:18:55 DEBUG 1713640735, 20:18:55 DEBUG ] 20:18:55 DEBUG "2844.0666666666666" 20:18:55 DEBUG } 20:18:55 DEBUG ] 20:18:55 DEBUG ] 20:18:55 DEBUG } 20:18:55 DEBUG } 20:18:55 DEBUG ] 20:18:55 DEBUG } 20:18:55 DEBUG } 20:18:55 DEBUG } 20:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640736" 20:18:56 INFO [http_cmd]: http status code OK 20:18:56 DEBUG --- status code --- 20:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:56 DEBUG --- http response --- 20:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:18:56 DEBUG ----- output ----- 20:18:56 DEBUG { 20:18:56 DEBUG "status": "success", 20:18:56 DEBUG "data": { 20:18:56 DEBUG "resultType": "vector", 20:18:56 DEBUG "result": [ 20:18:56 DEBUG { 20:18:56 DEBUG "metric": { 20:18:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:18:56 DEBUG }, 20:18:56 DEBUG "value": [ 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG [...] 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:18:56 DEBUG }, 20:18:56 DEBUG "value": [ 20:18:56 DEBUG 1713640736, 20:18:56 DEBUG "0" 20:18:56 DEBUG ] 20:18:56 DEBUG } 20:18:56 DEBUG ] 20:18:56 DEBUG } 20:18:56 DEBUG } 20:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640736" 20:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640736" 20:18:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640736" 20:18:56 INFO [http_cmd]: http status code OK 20:18:56 DEBUG --- status code --- 20:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:56 DEBUG --- http response --- 20:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:56 DEBUG ----- output ----- 20:18:56 DEBUG { 20:18:56 DEBUG "status": "success", 20:18:56 DEBUG "data": { 20:18:56 DEBUG "resultType": "vector", 20:18:56 DEBUG "result": [ 20:18:56 DEBUG { 20:18:56 DEBUG "metric": { 20:18:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:56 DEBUG }, 20:18:56 DEBUG "value": [ 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG [...] 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:56 DEBUG }, 20:18:56 DEBUG "value": [ 20:18:56 DEBUG 1713640736, 20:18:56 DEBUG "2832.233333333333" 20:18:56 DEBUG ] 20:18:56 DEBUG } 20:18:56 DEBUG ] 20:18:56 DEBUG } 20:18:56 DEBUG } 20:18:56 INFO [http_cmd]: http status code OK 20:18:56 DEBUG --- status code --- 20:18:56 INFO [http_cmd]: http status code OK 20:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:56 DEBUG --- status code --- 20:18:56 DEBUG --- http response --- 20:18:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:56 DEBUG --- http response --- 20:18:56 DEBUG ----- output ----- 20:18:56 DEBUG { 20:18:56 DEBUG "status": "success", 20:18:56 DEBUG "data": { 20:18:56 DEBUG "resultType": "vector", 20:18:56 DEBUG "result": [ 20:18:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:56 DEBUG ----- output ----- 20:18:56 DEBUG { 20:18:56 DEBUG "status": "success", 20:18:56 DEBUG "data": { 20:18:56 DEBUG "resultType": "vector", 20:18:56 DEBUG "result": [ 20:18:56 DEBUG { 20:18:56 DEBUG "metric": { 20:18:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:18:56 DEBUG }, 20:18:56 DEBUG "value": [ 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG [...] 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:18:56 DEBUG }, 20:18:56 DEBUG "value": [ 20:18:56 DEBUG 1713640736, 20:18:56 DEBUG "9.574034845819117" 20:18:56 DEBUG ] 20:18:56 DEBUG } 20:18:56 DEBUG ] 20:18:56 DEBUG } 20:18:56 DEBUG } 20:18:56 DEBUG { 20:18:56 DEBUG "metric": { 20:18:56 DEBUG "instance": "10.142.0.107:9100" 20:18:56 DEBUG }, 20:18:56 DEBUG "value": [ 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG [...] 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG 20:18:56 DEBUG "instance": "10.142.0.43:9100" 20:18:56 DEBUG }, 20:18:56 DEBUG "value": [ 20:18:56 DEBUG 1713640736, 20:18:56 DEBUG "3724664855552" 20:18:56 DEBUG ] 20:18:56 DEBUG } 20:18:56 DEBUG ] 20:18:56 DEBUG } 20:18:56 DEBUG } 20:18:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640737" 20:18:57 INFO [http_cmd]: http status code OK 20:18:57 DEBUG --- status code --- 20:18:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:57 DEBUG --- http response --- 20:18:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:57 DEBUG ----- output ----- 20:18:57 DEBUG { 20:18:57 DEBUG "status": "success", 20:18:57 DEBUG "data": { 20:18:57 DEBUG "resultType": "vector", 20:18:57 DEBUG "result": [ 20:18:57 DEBUG { 20:18:57 DEBUG "metric": { 20:18:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:57 DEBUG }, 20:18:57 DEBUG "value": [ 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG [...] 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:57 DEBUG }, 20:18:57 DEBUG "value": [ 20:18:57 DEBUG 1713640737, 20:18:57 DEBUG "2831.4333333333334" 20:18:57 DEBUG ] 20:18:57 DEBUG } 20:18:57 DEBUG ] 20:18:57 DEBUG } 20:18:57 DEBUG } 20:18:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640737" 20:18:57 INFO [http_cmd]: http status code OK 20:18:57 DEBUG --- status code --- 20:18:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:57 DEBUG --- http response --- 20:18:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:57 DEBUG ----- output ----- 20:18:57 DEBUG { 20:18:57 DEBUG "status": "success", 20:18:57 DEBUG "data": { 20:18:57 DEBUG "resultType": "vector", 20:18:57 DEBUG "result": [ 20:18:57 DEBUG { 20:18:57 DEBUG "metric": { 20:18:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:57 DEBUG }, 20:18:57 DEBUG "value": [ 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG [...] 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:57 DEBUG }, 20:18:57 DEBUG "value": [ 20:18:57 DEBUG 1713640737, 20:18:57 DEBUG "2831.4666666666667" 20:18:57 DEBUG ] 20:18:57 DEBUG } 20:18:57 DEBUG ] 20:18:57 DEBUG } 20:18:57 DEBUG } 20:18:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640737" 20:18:57 INFO [http_cmd]: http status code OK 20:18:57 DEBUG --- status code --- 20:18:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:57 DEBUG --- http response --- 20:18:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:18:57 DEBUG ----- output ----- 20:18:57 DEBUG { 20:18:57 DEBUG "status": "success", 20:18:57 DEBUG "data": { 20:18:57 DEBUG "resultType": "vector", 20:18:57 DEBUG "result": [ 20:18:57 DEBUG { 20:18:57 DEBUG "metric": { 20:18:57 DEBUG "pod": "ds-idrepo-1" 20:18:57 DEBUG }, 20:18:57 DEBUG "value": [ 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG [...] 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:18:57 DEBUG }, 20:18:57 DEBUG "value": [ 20:18:57 DEBUG 1713640737, 20:18:57 DEBUG "9.512802645425207" 20:18:57 DEBUG ] 20:18:57 DEBUG } 20:18:57 DEBUG ] 20:18:57 DEBUG } 20:18:57 DEBUG } 20:18:57 INFO 20:18:57 INFO [loop_until]: kubectl --namespace=xlou top pods 20:18:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:18:57 INFO [loop_until]: OK (rc = 0) 20:18:57 DEBUG --- stdout --- 20:18:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18441m 8662Mi am-767f6b67b5-qcqz2 18365m 9070Mi am-767f6b67b5-vbltz 19800m 8670Mi ds-cts-0 19854m 10988Mi ds-cts-1 18510m 10781Mi ds-cts-2 20323m 11056Mi ds-idrepo-0 8022m 3894Mi ds-idrepo-1 8016m 3883Mi ds-idrepo-2 8295m 3828Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9313m 836Mi 20:18:57 DEBUG --- stderr --- 20:18:57 DEBUG 20:18:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640737" 20:18:57 INFO [http_cmd]: http status code OK 20:18:57 DEBUG --- status code --- 20:18:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:57 DEBUG --- http response --- 20:18:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:57 DEBUG ----- output ----- 20:18:57 DEBUG { 20:18:57 DEBUG "status": "success", 20:18:57 DEBUG "data": { 20:18:57 DEBUG "resultType": "vector", 20:18:57 DEBUG "result": [ 20:18:57 DEBUG { 20:18:57 DEBUG "metric": { 20:18:57 DEBUG "pod": "ds-idrepo-0" 20:18:57 DEBUG }, 20:18:57 DEBUG "value": [ 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG [...] 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG 20:18:57 DEBUG "pod": "ds-idrepo-2" 20:18:57 DEBUG }, 20:18:57 DEBUG "value": [ 20:18:57 DEBUG 1713640737, 20:18:57 DEBUG "0" 20:18:57 DEBUG ] 20:18:57 DEBUG } 20:18:57 DEBUG ] 20:18:57 DEBUG } 20:18:57 DEBUG } 20:18:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640738" 20:18:58 INFO [http_cmd]: http status code OK 20:18:58 DEBUG --- status code --- 20:18:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:58 DEBUG --- http response --- 20:18:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:18:58 DEBUG ----- output ----- 20:18:58 DEBUG { 20:18:58 DEBUG "status": "success", 20:18:58 DEBUG "data": { 20:18:58 DEBUG "resultType": "vector", 20:18:58 DEBUG "result": [ 20:18:58 DEBUG { 20:18:58 DEBUG "metric": { 20:18:58 DEBUG "pod": "ds-cts-0" 20:18:58 DEBUG }, 20:18:58 DEBUG "value": [ 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG [...] 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG "pod": "ds-cts-2" 20:18:58 DEBUG }, 20:18:58 DEBUG "value": [ 20:18:58 DEBUG 1713640738, 20:18:58 DEBUG "0" 20:18:58 DEBUG ] 20:18:58 DEBUG } 20:18:58 DEBUG ] 20:18:58 DEBUG } 20:18:58 DEBUG } 20:18:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640738" 20:18:58 INFO [http_cmd]: http status code OK 20:18:58 DEBUG --- status code --- 20:18:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:58 DEBUG --- http response --- 20:18:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:18:58 DEBUG ----- output ----- 20:18:58 DEBUG { 20:18:58 DEBUG "status": "success", 20:18:58 DEBUG "data": { 20:18:58 DEBUG "resultType": "vector", 20:18:58 DEBUG "result": [ 20:18:58 DEBUG { 20:18:58 DEBUG "metric": { 20:18:58 DEBUG "pod": "ds-idrepo-0" 20:18:58 DEBUG }, 20:18:58 DEBUG "value": [ 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG [...] 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG "pod": "ds-cts-2" 20:18:58 DEBUG }, 20:18:58 DEBUG "value": [ 20:18:58 DEBUG 1713640738, 20:18:58 DEBUG "0" 20:18:58 DEBUG ] 20:18:58 DEBUG } 20:18:58 DEBUG ] 20:18:58 DEBUG } 20:18:58 DEBUG } 20:18:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640738" 20:18:58 INFO [http_cmd]: http status code OK 20:18:58 DEBUG --- status code --- 20:18:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:58 DEBUG --- http response --- 20:18:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:58 DEBUG ----- output ----- 20:18:58 DEBUG { 20:18:58 DEBUG "status": "success", 20:18:58 DEBUG "data": { 20:18:58 DEBUG "resultType": "vector", 20:18:58 DEBUG "result": [ 20:18:58 DEBUG { 20:18:58 DEBUG "metric": { 20:18:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:58 DEBUG }, 20:18:58 DEBUG "value": [ 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG [...] 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:58 DEBUG }, 20:18:58 DEBUG "value": [ 20:18:58 DEBUG 1713640738, 20:18:58 DEBUG "510166" 20:18:58 DEBUG ] 20:18:58 DEBUG } 20:18:58 DEBUG ] 20:18:58 DEBUG } 20:18:58 DEBUG } 20:18:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640738" 20:18:58 INFO [http_cmd]: http status code OK 20:18:58 DEBUG --- status code --- 20:18:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:58 DEBUG --- http response --- 20:18:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:58 DEBUG ----- output ----- 20:18:58 DEBUG { 20:18:58 DEBUG "status": "success", 20:18:58 DEBUG "data": { 20:18:58 DEBUG "resultType": "vector", 20:18:58 DEBUG "result": [ 20:18:58 DEBUG { 20:18:58 DEBUG "metric": { 20:18:58 DEBUG "instance": "10.142.0.107:9100" 20:18:58 DEBUG }, 20:18:58 DEBUG "value": [ 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG [...] 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG 20:18:58 DEBUG "instance": "10.142.0.43:9100" 20:18:58 DEBUG }, 20:18:58 DEBUG "value": [ 20:18:58 DEBUG 1713640738, 20:18:58 DEBUG "0.29186666666672734" 20:18:58 DEBUG ] 20:18:58 DEBUG } 20:18:58 DEBUG ] 20:18:58 DEBUG } 20:18:58 DEBUG } 20:18:59 INFO 20:18:59 INFO [loop_until]: kubectl --namespace=xlou top node 20:18:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640739" 20:18:59 INFO [http_cmd]: http status code OK 20:18:59 DEBUG --- status code --- 20:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:59 DEBUG --- http response --- 20:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:59 DEBUG ----- output ----- 20:18:59 DEBUG { 20:18:59 DEBUG "status": "success", 20:18:59 DEBUG "data": { 20:18:59 DEBUG "resultType": "vector", 20:18:59 DEBUG "result": [ 20:18:59 DEBUG { 20:18:59 DEBUG "metric": { 20:18:59 DEBUG "instance": "10.142.0.107:9100" 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG [...] 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG "instance": "10.142.0.43:9100" 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 1713640739, 20:18:59 DEBUG "85820583.36666667" 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG } 20:18:59 INFO [loop_until]: OK (rc = 0) 20:18:59 DEBUG --- stdout --- 20:18:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6223Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8184m 27% 7647Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18246m 61% 11374Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3328Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19835m 66% 14166Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5218Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18532m 62% 11960Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15689m 52% 2974Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20779m 69% 10893Mi 9% gke-xlou-cdm-ds-0f601751-j488 8735m 29% 7223Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18345m 61% 15824Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8640m 28% 7850Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20046m 67% 13713Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9447m 59% 6394Mi 10% 20:18:59 DEBUG --- stderr --- 20:18:59 DEBUG 20:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640739" 20:18:59 INFO [http_cmd]: http status code OK 20:18:59 DEBUG --- status code --- 20:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:59 DEBUG --- http response --- 20:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:59 DEBUG ----- output ----- 20:18:59 DEBUG { 20:18:59 DEBUG "status": "success", 20:18:59 DEBUG "data": { 20:18:59 DEBUG "resultType": "vector", 20:18:59 DEBUG "result": [ 20:18:59 DEBUG { 20:18:59 DEBUG "metric": { 20:18:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG [...] 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 1713640739, 20:18:59 DEBUG "971513856" 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG } 20:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640739" 20:18:59 INFO [http_cmd]: http status code OK 20:18:59 DEBUG --- status code --- 20:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:59 DEBUG --- http response --- 20:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:59 DEBUG ----- output ----- 20:18:59 DEBUG { 20:18:59 DEBUG "status": "success", 20:18:59 DEBUG "data": { 20:18:59 DEBUG "resultType": "vector", 20:18:59 DEBUG "result": [ 20:18:59 DEBUG { 20:18:59 DEBUG "metric": { 20:18:59 DEBUG "instance": "10.142.0.107:9100" 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG [...] 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG "instance": "10.142.0.43:9100" 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 1713640739, 20:18:59 DEBUG "85824210.86666667" 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG } 20:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640739" 20:18:59 INFO [http_cmd]: http status code OK 20:18:59 DEBUG --- status code --- 20:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:59 DEBUG --- http response --- 20:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:18:59 DEBUG ----- output ----- 20:18:59 DEBUG { 20:18:59 DEBUG "status": "success", 20:18:59 DEBUG "data": { 20:18:59 DEBUG "resultType": "vector", 20:18:59 DEBUG "result": [ 20:18:59 DEBUG { 20:18:59 DEBUG "metric": { 20:18:59 DEBUG "pod": "ds-idrepo-1" 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG [...] 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 1713640739, 20:18:59 DEBUG "62937.83882662054" 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG } 20:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640739" 20:18:59 INFO [http_cmd]: http status code OK 20:18:59 DEBUG --- status code --- 20:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:59 DEBUG --- http response --- 20:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:59 DEBUG ----- output ----- 20:18:59 DEBUG { 20:18:59 DEBUG "status": "success", 20:18:59 DEBUG "data": { 20:18:59 DEBUG "resultType": "vector", 20:18:59 DEBUG "result": [ 20:18:59 DEBUG { 20:18:59 DEBUG "metric": { 20:18:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG [...] 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 1713640739, 20:18:59 DEBUG "0" 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG } 20:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640739" 20:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640739" 20:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640739" 20:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640739" 20:18:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640739" 20:18:59 INFO [http_cmd]: http status code OK 20:18:59 INFO [http_cmd]: http status code OK 20:18:59 DEBUG --- status code --- 20:18:59 DEBUG --- status code --- 20:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:59 DEBUG --- http response --- 20:18:59 DEBUG --- http response --- 20:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:18:59 DEBUG ----- output ----- 20:18:59 DEBUG { 20:18:59 DEBUG "status": "success", 20:18:59 DEBUG "data": { 20:18:59 DEBUG "resultType": "vector", 20:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:59 DEBUG "result": [ 20:18:59 INFO [http_cmd]: http status code OK 20:18:59 DEBUG ----- output ----- 20:18:59 DEBUG { 20:18:59 DEBUG --- status code --- 20:18:59 DEBUG { 20:18:59 DEBUG "status": "success", 20:18:59 INFO [http_cmd]: http status code OK 20:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:59 DEBUG "metric": { 20:18:59 DEBUG "data": { 20:18:59 DEBUG --- status code --- 20:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:59 INFO [http_cmd]: http status code OK 20:18:59 DEBUG "pod": "ds-cts-0" 20:18:59 DEBUG "resultType": "vector", 20:18:59 DEBUG --- http response --- 20:18:59 DEBUG --- status code --- 20:18:59 DEBUG --- http response --- 20:18:59 DEBUG }, 20:18:59 DEBUG "result": [ 20:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:18:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:18:59 DEBUG "value": [ 20:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:18:59 DEBUG { 20:18:59 DEBUG ----- output ----- 20:18:59 DEBUG --- http response --- 20:18:59 DEBUG 20:18:59 DEBUG ----- output ----- 20:18:59 DEBUG "metric": { 20:18:59 DEBUG { 20:18:59 DEBUG 20:18:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:18:59 DEBUG { 20:18:59 DEBUG "pod": "ds-idrepo-1" 20:18:59 DEBUG "status": "success", 20:18:59 DEBUG 20:18:59 DEBUG ----- output ----- 20:18:59 DEBUG "status": "success", 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG [...] 20:18:59 DEBUG { 20:18:59 DEBUG "data": { 20:18:59 DEBUG 20:18:59 DEBUG "data": { 20:18:59 DEBUG 20:18:59 DEBUG "status": "success", 20:18:59 DEBUG "resultType": "vector", 20:18:59 DEBUG 20:18:59 DEBUG "resultType": "vector", 20:18:59 DEBUG 20:18:59 DEBUG "data": { 20:18:59 DEBUG "result": [ 20:18:59 DEBUG 20:18:59 DEBUG "result": [ 20:18:59 DEBUG 20:18:59 DEBUG "resultType": "vector", 20:18:59 DEBUG { 20:18:59 DEBUG [...] 20:18:59 DEBUG { 20:18:59 DEBUG "pod": "ds-cts-2" 20:18:59 DEBUG "result": [ 20:18:59 DEBUG "metric": { 20:18:59 DEBUG 20:18:59 DEBUG "metric": { 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:18:59 DEBUG 20:18:59 DEBUG "instance": "10.142.0.107:9100" 20:18:59 DEBUG 1713640739, 20:18:59 DEBUG { 20:18:59 DEBUG }, 20:18:59 DEBUG 20:18:59 DEBUG }, 20:18:59 DEBUG "0.0357" 20:18:59 DEBUG "metric": { 20:18:59 DEBUG "value": [ 20:18:59 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:18:59 DEBUG "value": [ 20:18:59 DEBUG ] 20:18:59 DEBUG "pod": "ds-cts-0" 20:18:59 DEBUG 20:18:59 DEBUG }, 20:18:59 DEBUG 20:18:59 DEBUG } 20:18:59 DEBUG }, 20:18:59 DEBUG 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 20:18:59 DEBUG } 20:18:59 DEBUG [...] 20:18:59 DEBUG 1713640739, 20:18:59 DEBUG 20:18:59 DEBUG [...] 20:18:59 DEBUG 20:18:59 DEBUG "0" 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:18:59 DEBUG ] 20:18:59 DEBUG [...] 20:18:59 DEBUG 20:18:59 DEBUG }, 20:18:59 DEBUG } 20:18:59 DEBUG 20:18:59 DEBUG "instance": "10.142.0.43:9100" 20:18:59 DEBUG "value": [ 20:18:59 DEBUG ] 20:18:59 DEBUG 20:18:59 DEBUG 20:18:59 DEBUG 1713640739, 20:18:59 DEBUG } 20:18:59 DEBUG "pod": "ds-cts-2" 20:18:59 DEBUG }, 20:18:59 DEBUG "0.1575" 20:18:59 DEBUG ] 20:18:59 DEBUG }, 20:18:59 DEBUG "value": [ 20:18:59 DEBUG } 20:18:59 DEBUG } 20:18:59 DEBUG "value": [ 20:18:59 DEBUG 1713640739, 20:18:59 DEBUG "30.933333333333334" 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG } 20:18:59 DEBUG 1713640739, 20:18:59 DEBUG "0.08933333333333356" 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG } 20:18:59 DEBUG ] 20:18:59 DEBUG } 20:18:59 DEBUG } 20:19:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640794" 20:19:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640794" 20:19:54 ERROR [http_cmd]: ERROR 20:19:54 INFO [http_cmd]: http status code OK 20:19:54 ERROR --- status code --- 20:19:54 DEBUG --- status code --- 20:19:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:19:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:54 ERROR --- http response --- 20:19:54 DEBUG --- http response --- 20:19:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:19:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:19:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640795" 20:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640795" 20:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640795" 20:19:55 INFO [http_cmd]: http status code OK 20:19:55 DEBUG --- status code --- 20:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:55 DEBUG --- http response --- 20:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:55 DEBUG ----- output ----- 20:19:55 DEBUG { 20:19:55 DEBUG "status": "success", 20:19:55 DEBUG "data": { 20:19:55 DEBUG "resultType": "vector", 20:19:55 DEBUG "result": [ 20:19:55 DEBUG { 20:19:55 DEBUG "metric": { 20:19:55 DEBUG "pod": "ds-idrepo-0" 20:19:55 DEBUG }, 20:19:55 DEBUG "value": [ 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG [...] 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG "pod": "ds-idrepo-2" 20:19:55 DEBUG }, 20:19:55 DEBUG "value": [ 20:19:55 DEBUG 1713640795, 20:19:55 DEBUG "0" 20:19:55 DEBUG ] 20:19:55 DEBUG } 20:19:55 DEBUG ] 20:19:55 DEBUG } 20:19:55 DEBUG } 20:19:55 INFO [http_cmd]: http status code OK 20:19:55 INFO [http_cmd]: http status code OK 20:19:55 DEBUG --- status code --- 20:19:55 DEBUG --- status code --- 20:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:55 DEBUG --- http response --- 20:19:55 DEBUG --- http response --- 20:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:55 DEBUG ----- output ----- 20:19:55 DEBUG ----- output ----- 20:19:55 DEBUG { 20:19:55 DEBUG { 20:19:55 DEBUG "status": "success", 20:19:55 DEBUG "status": "success", 20:19:55 DEBUG "data": { 20:19:55 DEBUG "data": { 20:19:55 DEBUG "resultType": "vector", 20:19:55 DEBUG "resultType": "vector", 20:19:55 DEBUG "result": [ 20:19:55 DEBUG "result": [ 20:19:55 DEBUG { 20:19:55 DEBUG { 20:19:55 DEBUG "metric": { 20:19:55 DEBUG "metric": { 20:19:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:55 DEBUG "pod": "ds-idrepo-0" 20:19:55 DEBUG }, 20:19:55 DEBUG }, 20:19:55 DEBUG "value": [ 20:19:55 DEBUG "value": [ 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG [...] 20:19:55 DEBUG [...] 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG 20:19:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:55 DEBUG "pod": "ds-idrepo-2" 20:19:55 DEBUG }, 20:19:55 DEBUG }, 20:19:55 DEBUG "value": [ 20:19:55 DEBUG "value": [ 20:19:55 DEBUG 1713640795, 20:19:55 DEBUG 1713640795, 20:19:55 DEBUG "2832.6" 20:19:55 DEBUG "0" 20:19:55 DEBUG ] 20:19:55 DEBUG ] 20:19:55 DEBUG } 20:19:55 DEBUG } 20:19:55 DEBUG ] 20:19:55 DEBUG } 20:19:55 DEBUG } 20:19:55 DEBUG ] 20:19:55 DEBUG } 20:19:55 DEBUG } 20:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640796" 20:19:56 INFO [http_cmd]: http status code OK 20:19:56 DEBUG --- status code --- 20:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:56 DEBUG --- http response --- 20:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:56 DEBUG ----- output ----- 20:19:56 DEBUG { 20:19:56 DEBUG "status": "success", 20:19:56 DEBUG "data": { 20:19:56 DEBUG "resultType": "vector", 20:19:56 DEBUG "result": [ 20:19:56 DEBUG { 20:19:56 DEBUG "metric": { 20:19:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:19:56 DEBUG }, 20:19:56 DEBUG "value": [ 20:19:56 DEBUG 20:19:56 DEBUG 20:19:56 DEBUG 20:19:56 DEBUG [...] 20:19:56 DEBUG 20:19:56 DEBUG 20:19:56 DEBUG 20:19:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:19:56 DEBUG }, 20:19:56 DEBUG "value": [ 20:19:56 DEBUG 1713640796, 20:19:56 DEBUG "0" 20:19:56 DEBUG ] 20:19:56 DEBUG } 20:19:56 DEBUG ] 20:19:56 DEBUG } 20:19:56 DEBUG } 20:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640796" 20:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640796" 20:19:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640796" 20:19:56 INFO [http_cmd]: http status code OK 20:19:56 DEBUG --- status code --- 20:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:56 DEBUG --- http response --- 20:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:56 DEBUG ----- output ----- 20:19:56 DEBUG { 20:19:56 DEBUG "status": "success", 20:19:56 DEBUG "data": { 20:19:56 DEBUG "resultType": "vector", 20:19:56 DEBUG "result": [ 20:19:56 DEBUG { 20:19:56 DEBUG "metric": { 20:19:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:56 DEBUG }, 20:19:56 DEBUG "value": [ 20:19:56 DEBUG 20:19:56 DEBUG 20:19:56 DEBUG 20:19:56 DEBUG [...] 20:19:56 DEBUG 20:19:56 DEBUG 20:19:56 DEBUG 20:19:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:56 DEBUG }, 20:19:56 DEBUG "value": [ 20:19:56 DEBUG 1713640796, 20:19:56 DEBUG "2831.866666666667" 20:19:56 DEBUG ] 20:19:56 DEBUG } 20:19:56 DEBUG ] 20:19:56 DEBUG } 20:19:56 DEBUG } 20:19:56 INFO [http_cmd]: http status code OK 20:19:56 DEBUG --- status code --- 20:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:56 DEBUG --- http response --- 20:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:56 DEBUG ----- output ----- 20:19:56 DEBUG { 20:19:56 DEBUG "status": "success", 20:19:56 DEBUG "data": { 20:19:56 DEBUG "resultType": "vector", 20:19:56 DEBUG "result": [ 20:19:56 DEBUG { 20:19:56 INFO [http_cmd]: http status code OK 20:19:56 DEBUG "metric": { 20:19:56 DEBUG --- status code --- 20:19:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:19:56 DEBUG }, 20:19:56 DEBUG "value": [ 20:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:56 DEBUG 20:19:56 DEBUG --- http response --- 20:19:56 DEBUG 20:19:56 DEBUG 20:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:56 DEBUG [...] 20:19:56 DEBUG ----- output ----- 20:19:56 DEBUG 20:19:56 DEBUG { 20:19:56 DEBUG 20:19:56 DEBUG "status": "success", 20:19:56 DEBUG 20:19:56 DEBUG "data": { 20:19:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:19:56 DEBUG "resultType": "vector", 20:19:56 DEBUG }, 20:19:56 DEBUG "result": [ 20:19:56 DEBUG "value": [ 20:19:56 DEBUG { 20:19:56 DEBUG 1713640796, 20:19:56 DEBUG "metric": { 20:19:56 DEBUG "9.572253982157141" 20:19:56 DEBUG "instance": "10.142.0.107:9100" 20:19:56 DEBUG ] 20:19:56 DEBUG }, 20:19:56 DEBUG } 20:19:56 DEBUG "value": [ 20:19:56 DEBUG ] 20:19:56 DEBUG 20:19:56 DEBUG } 20:19:56 DEBUG 20:19:56 DEBUG } 20:19:56 DEBUG 20:19:56 DEBUG [...] 20:19:56 DEBUG 20:19:56 DEBUG 20:19:56 DEBUG 20:19:56 DEBUG "instance": "10.142.0.43:9100" 20:19:56 DEBUG }, 20:19:56 DEBUG "value": [ 20:19:56 DEBUG 1713640796, 20:19:56 DEBUG "3728829797376" 20:19:56 DEBUG ] 20:19:56 DEBUG } 20:19:56 DEBUG ] 20:19:56 DEBUG } 20:19:56 DEBUG } 20:19:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640797" 20:19:57 INFO [http_cmd]: http status code OK 20:19:57 DEBUG --- status code --- 20:19:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:57 DEBUG --- http response --- 20:19:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:57 DEBUG ----- output ----- 20:19:57 DEBUG { 20:19:57 DEBUG "status": "success", 20:19:57 DEBUG "data": { 20:19:57 DEBUG "resultType": "vector", 20:19:57 DEBUG "result": [ 20:19:57 DEBUG { 20:19:57 DEBUG "metric": { 20:19:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:57 DEBUG }, 20:19:57 DEBUG "value": [ 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG [...] 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:57 DEBUG }, 20:19:57 DEBUG "value": [ 20:19:57 DEBUG 1713640797, 20:19:57 DEBUG "2831.366666666667" 20:19:57 DEBUG ] 20:19:57 DEBUG } 20:19:57 DEBUG ] 20:19:57 DEBUG } 20:19:57 DEBUG } 20:19:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640797" 20:19:57 INFO [http_cmd]: http status code OK 20:19:57 DEBUG --- status code --- 20:19:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:57 DEBUG --- http response --- 20:19:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:57 DEBUG ----- output ----- 20:19:57 DEBUG { 20:19:57 DEBUG "status": "success", 20:19:57 DEBUG "data": { 20:19:57 DEBUG "resultType": "vector", 20:19:57 DEBUG "result": [ 20:19:57 DEBUG { 20:19:57 DEBUG "metric": { 20:19:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:57 DEBUG }, 20:19:57 DEBUG "value": [ 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG [...] 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:57 DEBUG }, 20:19:57 DEBUG "value": [ 20:19:57 DEBUG 1713640797, 20:19:57 DEBUG "2831.8" 20:19:57 DEBUG ] 20:19:57 DEBUG } 20:19:57 DEBUG ] 20:19:57 DEBUG } 20:19:57 DEBUG } 20:19:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640797" 20:19:57 INFO [http_cmd]: http status code OK 20:19:57 DEBUG --- status code --- 20:19:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:57 DEBUG --- http response --- 20:19:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:19:57 DEBUG ----- output ----- 20:19:57 DEBUG { 20:19:57 DEBUG "status": "success", 20:19:57 DEBUG "data": { 20:19:57 DEBUG "resultType": "vector", 20:19:57 DEBUG "result": [ 20:19:57 DEBUG { 20:19:57 DEBUG "metric": { 20:19:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:19:57 DEBUG }, 20:19:57 DEBUG "value": [ 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG [...] 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:19:57 DEBUG }, 20:19:57 DEBUG "value": [ 20:19:57 DEBUG 1713640797, 20:19:57 DEBUG "0.000014927748691101853" 20:19:57 DEBUG ] 20:19:57 DEBUG } 20:19:57 DEBUG ] 20:19:57 DEBUG } 20:19:57 DEBUG } 20:19:57 INFO 20:19:57 INFO [loop_until]: kubectl --namespace=xlou top pods 20:19:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:19:57 INFO [loop_until]: OK (rc = 0) 20:19:57 DEBUG --- stdout --- 20:19:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18308m 8660Mi am-767f6b67b5-qcqz2 18514m 9072Mi am-767f6b67b5-vbltz 19839m 8670Mi ds-cts-0 19588m 11051Mi ds-cts-1 18700m 10844Mi ds-cts-2 20162m 11082Mi ds-idrepo-0 7830m 3893Mi ds-idrepo-1 7807m 3882Mi ds-idrepo-2 7783m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9575m 838Mi 20:19:57 DEBUG --- stderr --- 20:19:57 DEBUG 20:19:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640797" 20:19:57 INFO [http_cmd]: http status code OK 20:19:57 DEBUG --- status code --- 20:19:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:57 DEBUG --- http response --- 20:19:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:57 DEBUG ----- output ----- 20:19:57 DEBUG { 20:19:57 DEBUG "status": "success", 20:19:57 DEBUG "data": { 20:19:57 DEBUG "resultType": "vector", 20:19:57 DEBUG "result": [ 20:19:57 DEBUG { 20:19:57 DEBUG "metric": { 20:19:57 DEBUG "pod": "ds-idrepo-0" 20:19:57 DEBUG }, 20:19:57 DEBUG "value": [ 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG [...] 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG 20:19:57 DEBUG "pod": "ds-idrepo-2" 20:19:57 DEBUG }, 20:19:57 DEBUG "value": [ 20:19:57 DEBUG 1713640797, 20:19:57 DEBUG "0" 20:19:57 DEBUG ] 20:19:57 DEBUG } 20:19:57 DEBUG ] 20:19:57 DEBUG } 20:19:57 DEBUG } 20:19:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640798" 20:19:58 INFO [http_cmd]: http status code OK 20:19:58 DEBUG --- status code --- 20:19:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:58 DEBUG --- http response --- 20:19:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:19:58 DEBUG ----- output ----- 20:19:58 DEBUG { 20:19:58 DEBUG "status": "success", 20:19:58 DEBUG "data": { 20:19:58 DEBUG "resultType": "vector", 20:19:58 DEBUG "result": [ 20:19:58 DEBUG { 20:19:58 DEBUG "metric": { 20:19:58 DEBUG "pod": "ds-cts-0" 20:19:58 DEBUG }, 20:19:58 DEBUG "value": [ 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG [...] 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG "pod": "ds-cts-2" 20:19:58 DEBUG }, 20:19:58 DEBUG "value": [ 20:19:58 DEBUG 1713640798, 20:19:58 DEBUG "0" 20:19:58 DEBUG ] 20:19:58 DEBUG } 20:19:58 DEBUG ] 20:19:58 DEBUG } 20:19:58 DEBUG } 20:19:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640798" 20:19:58 INFO [http_cmd]: http status code OK 20:19:58 DEBUG --- status code --- 20:19:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:58 DEBUG --- http response --- 20:19:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:19:58 DEBUG ----- output ----- 20:19:58 DEBUG { 20:19:58 DEBUG "status": "success", 20:19:58 DEBUG "data": { 20:19:58 DEBUG "resultType": "vector", 20:19:58 DEBUG "result": [ 20:19:58 DEBUG { 20:19:58 DEBUG "metric": { 20:19:58 DEBUG "pod": "ds-idrepo-0" 20:19:58 DEBUG }, 20:19:58 DEBUG "value": [ 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG [...] 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG "pod": "ds-cts-2" 20:19:58 DEBUG }, 20:19:58 DEBUG "value": [ 20:19:58 DEBUG 1713640798, 20:19:58 DEBUG "0" 20:19:58 DEBUG ] 20:19:58 DEBUG } 20:19:58 DEBUG ] 20:19:58 DEBUG } 20:19:58 DEBUG } 20:19:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640798" 20:19:58 INFO [http_cmd]: http status code OK 20:19:58 DEBUG --- status code --- 20:19:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:58 DEBUG --- http response --- 20:19:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:58 DEBUG ----- output ----- 20:19:58 DEBUG { 20:19:58 DEBUG "status": "success", 20:19:58 DEBUG "data": { 20:19:58 DEBUG "resultType": "vector", 20:19:58 DEBUG "result": [ 20:19:58 DEBUG { 20:19:58 DEBUG "metric": { 20:19:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:58 DEBUG }, 20:19:58 DEBUG "value": [ 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG [...] 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:58 DEBUG }, 20:19:58 DEBUG "value": [ 20:19:58 DEBUG 1713640798, 20:19:58 DEBUG "510123" 20:19:58 DEBUG ] 20:19:58 DEBUG } 20:19:58 DEBUG ] 20:19:58 DEBUG } 20:19:58 DEBUG } 20:19:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640798" 20:19:58 INFO [http_cmd]: http status code OK 20:19:58 DEBUG --- status code --- 20:19:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:58 DEBUG --- http response --- 20:19:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:58 DEBUG ----- output ----- 20:19:58 DEBUG { 20:19:58 DEBUG "status": "success", 20:19:58 DEBUG "data": { 20:19:58 DEBUG "resultType": "vector", 20:19:58 DEBUG "result": [ 20:19:58 DEBUG { 20:19:58 DEBUG "metric": { 20:19:58 DEBUG "instance": "10.142.0.107:9100" 20:19:58 DEBUG }, 20:19:58 DEBUG "value": [ 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG [...] 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG 20:19:58 DEBUG "instance": "10.142.0.43:9100" 20:19:58 DEBUG }, 20:19:58 DEBUG "value": [ 20:19:58 DEBUG 1713640798, 20:19:58 DEBUG "0.2072333333333167" 20:19:58 DEBUG ] 20:19:58 DEBUG } 20:19:58 DEBUG ] 20:19:58 DEBUG } 20:19:58 DEBUG } 20:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640799" 20:19:59 INFO 20:19:59 INFO [loop_until]: kubectl --namespace=xlou top node 20:19:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:19:59 INFO [http_cmd]: http status code OK 20:19:59 DEBUG --- status code --- 20:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:59 DEBUG --- http response --- 20:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:59 DEBUG ----- output ----- 20:19:59 DEBUG { 20:19:59 DEBUG "status": "success", 20:19:59 DEBUG "data": { 20:19:59 DEBUG "resultType": "vector", 20:19:59 DEBUG "result": [ 20:19:59 DEBUG { 20:19:59 DEBUG "metric": { 20:19:59 DEBUG "instance": "10.142.0.107:9100" 20:19:59 DEBUG }, 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG [...] 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG "instance": "10.142.0.43:9100" 20:19:59 DEBUG }, 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 1713640799, 20:19:59 DEBUG "85502537.8" 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG } 20:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640799" 20:19:59 INFO [http_cmd]: http status code OK 20:19:59 DEBUG --- status code --- 20:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:59 DEBUG --- http response --- 20:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:59 DEBUG ----- output ----- 20:19:59 DEBUG { 20:19:59 DEBUG "status": "success", 20:19:59 DEBUG "data": { 20:19:59 DEBUG "resultType": "vector", 20:19:59 DEBUG "result": [ 20:19:59 DEBUG { 20:19:59 DEBUG "metric": { 20:19:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:19:59 DEBUG }, 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG [...] 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:19:59 DEBUG }, 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 1713640799, 20:19:59 DEBUG "976498688" 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG } 20:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640799" 20:19:59 INFO [loop_until]: OK (rc = 0) 20:19:59 DEBUG --- stdout --- 20:19:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6217Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8167m 27% 7639Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18361m 61% 11381Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3319Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19833m 66% 14225Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5193Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18563m 62% 11955Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15752m 52% 2968Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20865m 69% 10896Mi 9% gke-xlou-cdm-ds-0f601751-j488 8750m 29% 7230Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18740m 62% 15886Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8623m 28% 7850Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20269m 67% 13730Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9740m 61% 6394Mi 10% 20:19:59 DEBUG --- stderr --- 20:19:59 DEBUG 20:19:59 INFO [http_cmd]: http status code OK 20:19:59 DEBUG --- status code --- 20:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:59 DEBUG --- http response --- 20:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:59 DEBUG ----- output ----- 20:19:59 DEBUG { 20:19:59 DEBUG "status": "success", 20:19:59 DEBUG "data": { 20:19:59 DEBUG "resultType": "vector", 20:19:59 DEBUG "result": [ 20:19:59 DEBUG { 20:19:59 DEBUG "metric": { 20:19:59 DEBUG "instance": "10.142.0.107:9100" 20:19:59 DEBUG }, 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG [...] 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG "instance": "10.142.0.43:9100" 20:19:59 DEBUG }, 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 1713640799, 20:19:59 DEBUG "85505997.1" 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG } 20:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640799" 20:19:59 INFO [http_cmd]: http status code OK 20:19:59 DEBUG --- status code --- 20:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:59 DEBUG --- http response --- 20:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:59 DEBUG ----- output ----- 20:19:59 DEBUG { 20:19:59 DEBUG "status": "success", 20:19:59 DEBUG "data": { 20:19:59 DEBUG "resultType": "vector", 20:19:59 DEBUG "result": [ 20:19:59 DEBUG { 20:19:59 DEBUG "metric": { 20:19:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:19:59 DEBUG }, 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG [...] 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:19:59 DEBUG }, 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 1713640799, 20:19:59 DEBUG "0" 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG } 20:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640799" 20:19:59 INFO [http_cmd]: http status code OK 20:19:59 DEBUG --- status code --- 20:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:59 DEBUG --- http response --- 20:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:59 DEBUG ----- output ----- 20:19:59 DEBUG { 20:19:59 DEBUG "status": "success", 20:19:59 DEBUG "data": { 20:19:59 DEBUG "resultType": "vector", 20:19:59 DEBUG "result": [ 20:19:59 DEBUG { 20:19:59 DEBUG "metric": { 20:19:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:59 DEBUG }, 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG [...] 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:59 DEBUG }, 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 1713640799, 20:19:59 DEBUG "0" 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG } 20:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640799" 20:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640799" 20:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640799" 20:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640799" 20:19:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640799" 20:19:59 INFO [http_cmd]: http status code OK 20:19:59 INFO [http_cmd]: http status code OK 20:19:59 DEBUG --- status code --- 20:19:59 DEBUG --- status code --- 20:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:59 DEBUG --- http response --- 20:19:59 DEBUG --- http response --- 20:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:19:59 INFO [http_cmd]: http status code OK 20:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:19:59 DEBUG ----- output ----- 20:19:59 DEBUG --- status code --- 20:19:59 DEBUG ----- output ----- 20:19:59 DEBUG { 20:19:59 INFO [http_cmd]: http status code OK 20:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:59 DEBUG { 20:19:59 DEBUG "status": "success", 20:19:59 DEBUG --- status code --- 20:19:59 DEBUG --- http response --- 20:19:59 DEBUG "status": "success", 20:19:59 DEBUG "data": { 20:19:59 DEBUG "resultType": "vector", 20:19:59 DEBUG "data": { 20:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:59 DEBUG "result": [ 20:19:59 INFO [http_cmd]: http status code OK 20:19:59 DEBUG "resultType": "vector", 20:19:59 DEBUG ----- output ----- 20:19:59 DEBUG --- http response --- 20:19:59 DEBUG { 20:19:59 DEBUG --- status code --- 20:19:59 DEBUG "result": [ 20:19:59 DEBUG { 20:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:19:59 DEBUG "metric": { 20:19:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:19:59 DEBUG { 20:19:59 DEBUG "status": "success", 20:19:59 DEBUG "data": { 20:19:59 DEBUG ----- output ----- 20:19:59 DEBUG --- http response --- 20:19:59 DEBUG "metric": { 20:19:59 DEBUG "pod": "ds-cts-0" 20:19:59 DEBUG "resultType": "vector", 20:19:59 DEBUG { 20:19:59 DEBUG "pod": "ds-cts-0" 20:19:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:19:59 DEBUG }, 20:19:59 DEBUG "result": [ 20:19:59 DEBUG "status": "success", 20:19:59 DEBUG }, 20:19:59 DEBUG ----- output ----- 20:19:59 DEBUG "value": [ 20:19:59 DEBUG { 20:19:59 DEBUG "data": { 20:19:59 DEBUG "value": [ 20:19:59 DEBUG { 20:19:59 DEBUG 20:19:59 DEBUG "metric": { 20:19:59 DEBUG "resultType": "vector", 20:19:59 DEBUG 20:19:59 DEBUG "status": "success", 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG "result": [ 20:19:59 DEBUG 20:19:59 DEBUG "data": { 20:19:59 DEBUG [...] 20:19:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:19:59 DEBUG { 20:19:59 DEBUG 20:19:59 DEBUG "resultType": "vector", 20:19:59 DEBUG 20:19:59 DEBUG }, 20:19:59 DEBUG "metric": { 20:19:59 DEBUG [...] 20:19:59 DEBUG "result": [ 20:19:59 DEBUG 20:19:59 DEBUG "value": [ 20:19:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:19:59 DEBUG 20:19:59 DEBUG { 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG }, 20:19:59 DEBUG 20:19:59 DEBUG "metric": { 20:19:59 DEBUG "pod": "ds-cts-2" 20:19:59 DEBUG 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 20:19:59 DEBUG "instance": "10.142.0.107:9100" 20:19:59 DEBUG }, 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG "pod": "ds-cts-2" 20:19:59 DEBUG }, 20:19:59 DEBUG "value": [ 20:19:59 DEBUG [...] 20:19:59 DEBUG 20:19:59 DEBUG }, 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 1713640799, 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 20:19:59 DEBUG "0.030733333333333324" 20:19:59 DEBUG 20:19:59 DEBUG [...] 20:19:59 DEBUG 1713640799, 20:19:59 DEBUG 20:19:59 DEBUG ] 20:19:59 DEBUG 20:19:59 DEBUG 20:19:59 DEBUG "30.9" 20:19:59 DEBUG 20:19:59 DEBUG } 20:19:59 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:19:59 DEBUG }, 20:19:59 DEBUG ] 20:19:59 DEBUG [...] 20:19:59 DEBUG ] 20:19:59 DEBUG 20:19:59 DEBUG "value": [ 20:19:59 DEBUG } 20:19:59 DEBUG 20:19:59 DEBUG } 20:19:59 DEBUG 20:19:59 DEBUG 1713640799, 20:19:59 DEBUG ] 20:19:59 DEBUG 20:19:59 DEBUG } 20:19:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:19:59 DEBUG "0" 20:19:59 DEBUG } 20:19:59 DEBUG 20:19:59 DEBUG }, 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG "instance": "10.142.0.43:9100" 20:19:59 DEBUG "value": [ 20:19:59 DEBUG } 20:19:59 DEBUG }, 20:19:59 DEBUG 1713640799, 20:19:59 DEBUG ] 20:19:59 DEBUG "value": [ 20:19:59 DEBUG 1713640799, 20:19:59 DEBUG } 20:19:59 DEBUG "0.06733333333331568" 20:19:59 DEBUG "0.159" 20:19:59 DEBUG } 20:19:59 DEBUG ] 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG } 20:19:59 DEBUG ] 20:19:59 DEBUG ] 20:19:59 DEBUG } 20:19:59 DEBUG } 20:19:59 DEBUG } 20:19:59 DEBUG } 20:20:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640854" 20:20:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640854" 20:20:54 ERROR [http_cmd]: ERROR 20:20:54 ERROR --- status code --- 20:20:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:20:54 ERROR --- http response --- 20:20:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:20:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:20:54 INFO [http_cmd]: http status code OK 20:20:54 DEBUG --- status code --- 20:20:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:54 DEBUG --- http response --- 20:20:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:20:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640855" 20:20:55 INFO [http_cmd]: http status code OK 20:20:55 DEBUG --- status code --- 20:20:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:55 DEBUG --- http response --- 20:20:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:55 DEBUG ----- output ----- 20:20:55 DEBUG { 20:20:55 DEBUG "status": "success", 20:20:55 DEBUG "data": { 20:20:55 DEBUG "resultType": "vector", 20:20:55 DEBUG "result": [ 20:20:55 DEBUG { 20:20:55 DEBUG "metric": { 20:20:55 DEBUG "pod": "ds-idrepo-0" 20:20:55 DEBUG }, 20:20:55 DEBUG "value": [ 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG [...] 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG "pod": "ds-idrepo-2" 20:20:55 DEBUG }, 20:20:55 DEBUG "value": [ 20:20:55 DEBUG 1713640855, 20:20:55 DEBUG "0" 20:20:55 DEBUG ] 20:20:55 DEBUG } 20:20:55 DEBUG ] 20:20:55 DEBUG } 20:20:55 DEBUG } 20:20:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640855" 20:20:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640855" 20:20:55 INFO [http_cmd]: http status code OK 20:20:55 DEBUG --- status code --- 20:20:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:55 DEBUG --- http response --- 20:20:55 INFO [http_cmd]: http status code OK 20:20:55 DEBUG --- status code --- 20:20:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:55 DEBUG --- http response --- 20:20:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:55 DEBUG ----- output ----- 20:20:55 DEBUG ----- output ----- 20:20:55 DEBUG { 20:20:55 DEBUG { 20:20:55 DEBUG "status": "success", 20:20:55 DEBUG "status": "success", 20:20:55 DEBUG "data": { 20:20:55 DEBUG "data": { 20:20:55 DEBUG "resultType": "vector", 20:20:55 DEBUG "resultType": "vector", 20:20:55 DEBUG "result": [ 20:20:55 DEBUG "result": [ 20:20:55 DEBUG { 20:20:55 DEBUG { 20:20:55 DEBUG "metric": { 20:20:55 DEBUG "metric": { 20:20:55 DEBUG "pod": "ds-idrepo-0" 20:20:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:55 DEBUG }, 20:20:55 DEBUG }, 20:20:55 DEBUG "value": [ 20:20:55 DEBUG "value": [ 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG [...] 20:20:55 DEBUG [...] 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:55 DEBUG }, 20:20:55 DEBUG "value": [ 20:20:55 DEBUG 1713640855, 20:20:55 DEBUG "2822.5666666666666" 20:20:55 DEBUG ] 20:20:55 DEBUG } 20:20:55 DEBUG ] 20:20:55 DEBUG } 20:20:55 DEBUG } 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG 20:20:55 DEBUG "pod": "ds-idrepo-2" 20:20:55 DEBUG }, 20:20:55 DEBUG "value": [ 20:20:55 DEBUG 1713640855, 20:20:55 DEBUG "0" 20:20:55 DEBUG ] 20:20:55 DEBUG } 20:20:55 DEBUG ] 20:20:55 DEBUG } 20:20:55 DEBUG } 20:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640856" 20:20:56 INFO [http_cmd]: http status code OK 20:20:56 DEBUG --- status code --- 20:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:56 DEBUG --- http response --- 20:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:56 DEBUG ----- output ----- 20:20:56 DEBUG { 20:20:56 DEBUG "status": "success", 20:20:56 DEBUG "data": { 20:20:56 DEBUG "resultType": "vector", 20:20:56 DEBUG "result": [ 20:20:56 DEBUG { 20:20:56 DEBUG "metric": { 20:20:56 DEBUG "pod": "ds-idrepo-1" 20:20:56 DEBUG }, 20:20:56 DEBUG "value": [ 20:20:56 DEBUG 20:20:56 DEBUG 20:20:56 DEBUG 20:20:56 DEBUG [...] 20:20:56 DEBUG 20:20:56 DEBUG 20:20:56 DEBUG 20:20:56 DEBUG "pod": "ds-cts-0" 20:20:56 DEBUG }, 20:20:56 DEBUG "value": [ 20:20:56 DEBUG 1713640856, 20:20:56 DEBUG "43422555.28661257" 20:20:56 DEBUG ] 20:20:56 DEBUG } 20:20:56 DEBUG ] 20:20:56 DEBUG } 20:20:56 DEBUG } 20:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640856" 20:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640856" 20:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640856" 20:20:56 INFO [http_cmd]: http status code OK 20:20:56 DEBUG --- status code --- 20:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:56 DEBUG --- http response --- 20:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:56 DEBUG ----- output ----- 20:20:56 DEBUG { 20:20:56 DEBUG "status": "success", 20:20:56 DEBUG "data": { 20:20:56 DEBUG "resultType": "vector", 20:20:56 DEBUG "result": [ 20:20:56 DEBUG { 20:20:56 DEBUG "metric": { 20:20:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:56 DEBUG }, 20:20:56 DEBUG "value": [ 20:20:56 DEBUG 20:20:56 DEBUG 20:20:56 DEBUG 20:20:56 DEBUG [...] 20:20:56 DEBUG 20:20:56 DEBUG 20:20:56 DEBUG 20:20:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:56 DEBUG }, 20:20:56 DEBUG "value": [ 20:20:56 DEBUG 1713640856, 20:20:56 DEBUG "2844.866666666667" 20:20:56 DEBUG ] 20:20:56 DEBUG } 20:20:56 DEBUG ] 20:20:56 DEBUG } 20:20:56 DEBUG } 20:20:56 INFO [http_cmd]: http status code OK 20:20:56 DEBUG --- status code --- 20:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:56 DEBUG --- http response --- 20:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:56 DEBUG ----- output ----- 20:20:56 DEBUG { 20:20:56 DEBUG "status": "success", 20:20:56 INFO [http_cmd]: http status code OK 20:20:56 DEBUG "data": { 20:20:56 DEBUG --- status code --- 20:20:56 DEBUG "resultType": "vector", 20:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:56 DEBUG "result": [ 20:20:56 DEBUG --- http response --- 20:20:56 DEBUG { 20:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:56 DEBUG "metric": { 20:20:56 DEBUG ----- output ----- 20:20:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:20:56 DEBUG { 20:20:56 DEBUG }, 20:20:56 DEBUG "status": "success", 20:20:56 DEBUG "value": [ 20:20:56 DEBUG "data": { 20:20:56 DEBUG 20:20:56 DEBUG "resultType": "vector", 20:20:56 DEBUG 20:20:56 DEBUG "result": [ 20:20:56 DEBUG 20:20:56 DEBUG { 20:20:56 DEBUG [...] 20:20:56 DEBUG "metric": { 20:20:56 DEBUG 20:20:56 DEBUG "instance": "10.142.0.107:9100" 20:20:56 DEBUG 20:20:56 DEBUG }, 20:20:56 DEBUG 20:20:56 DEBUG "value": [ 20:20:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:20:56 DEBUG 20:20:56 DEBUG }, 20:20:56 DEBUG 20:20:56 DEBUG "value": [ 20:20:56 DEBUG 20:20:56 DEBUG 1713640856, 20:20:56 DEBUG [...] 20:20:56 DEBUG "9.561535813792743" 20:20:56 DEBUG 20:20:56 DEBUG 20:20:56 DEBUG 20:20:56 DEBUG "instance": "10.142.0.43:9100" 20:20:56 DEBUG }, 20:20:56 DEBUG "value": [ 20:20:56 DEBUG 1713640856, 20:20:56 DEBUG "3733464402944" 20:20:56 DEBUG ] 20:20:56 DEBUG } 20:20:56 DEBUG ] 20:20:56 DEBUG } 20:20:56 DEBUG } 20:20:56 DEBUG ] 20:20:56 DEBUG } 20:20:56 DEBUG ] 20:20:56 DEBUG } 20:20:56 DEBUG } 20:20:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640857" 20:20:57 INFO [http_cmd]: http status code OK 20:20:57 DEBUG --- status code --- 20:20:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:57 DEBUG --- http response --- 20:20:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:57 DEBUG ----- output ----- 20:20:57 DEBUG { 20:20:57 DEBUG "status": "success", 20:20:57 DEBUG "data": { 20:20:57 DEBUG "resultType": "vector", 20:20:57 DEBUG "result": [ 20:20:57 DEBUG { 20:20:57 DEBUG "metric": { 20:20:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:57 DEBUG }, 20:20:57 DEBUG "value": [ 20:20:57 DEBUG 20:20:57 DEBUG 20:20:57 DEBUG 20:20:57 DEBUG [...] 20:20:57 DEBUG 20:20:57 DEBUG 20:20:57 DEBUG 20:20:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:57 DEBUG }, 20:20:57 DEBUG "value": [ 20:20:57 DEBUG 1713640857, 20:20:57 DEBUG "2844.6" 20:20:57 DEBUG ] 20:20:57 DEBUG } 20:20:57 DEBUG ] 20:20:57 DEBUG } 20:20:57 DEBUG } 20:20:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640857" 20:20:57 INFO [http_cmd]: http status code OK 20:20:57 DEBUG --- status code --- 20:20:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:57 DEBUG --- http response --- 20:20:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:57 DEBUG ----- output ----- 20:20:57 DEBUG { 20:20:57 DEBUG "status": "success", 20:20:57 DEBUG "data": { 20:20:57 DEBUG "resultType": "vector", 20:20:57 DEBUG "result": [ 20:20:57 DEBUG { 20:20:57 DEBUG "metric": { 20:20:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:57 DEBUG }, 20:20:57 DEBUG "value": [ 20:20:57 DEBUG 20:20:57 DEBUG 20:20:57 DEBUG 20:20:57 DEBUG [...] 20:20:57 DEBUG 20:20:57 DEBUG 20:20:57 DEBUG 20:20:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:57 DEBUG }, 20:20:57 DEBUG "value": [ 20:20:57 DEBUG 1713640857, 20:20:57 DEBUG "2842" 20:20:57 DEBUG ] 20:20:57 DEBUG } 20:20:57 DEBUG ] 20:20:57 DEBUG } 20:20:57 DEBUG } 20:20:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640857" 20:20:57 INFO [http_cmd]: http status code OK 20:20:57 DEBUG --- status code --- 20:20:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:57 DEBUG --- http response --- 20:20:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:57 DEBUG ----- output ----- 20:20:57 DEBUG { 20:20:57 DEBUG "status": "success", 20:20:57 DEBUG "data": { 20:20:57 DEBUG "resultType": "vector", 20:20:57 DEBUG "result": [ 20:20:57 DEBUG { 20:20:57 DEBUG "metric": { 20:20:57 DEBUG "pod": "ds-idrepo-1" 20:20:57 DEBUG }, 20:20:57 DEBUG "value": [ 20:20:57 DEBUG 20:20:57 DEBUG 20:20:57 DEBUG 20:20:57 DEBUG [...] 20:20:57 DEBUG 20:20:57 DEBUG 20:20:57 DEBUG 20:20:57 DEBUG "pod": "ds-cts-0" 20:20:57 DEBUG }, 20:20:57 DEBUG "value": [ 20:20:57 DEBUG 1713640857, 20:20:57 DEBUG "10.378232651315178" 20:20:57 DEBUG ] 20:20:57 DEBUG } 20:20:57 DEBUG ] 20:20:57 DEBUG } 20:20:57 DEBUG } 20:20:57 INFO 20:20:57 INFO [loop_until]: kubectl --namespace=xlou top pods 20:20:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:20:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640858" 20:20:58 INFO [loop_until]: OK (rc = 0) 20:20:58 DEBUG --- stdout --- 20:20:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18709m 8661Mi am-767f6b67b5-qcqz2 18376m 9074Mi am-767f6b67b5-vbltz 21112m 8671Mi ds-cts-0 19875m 11047Mi ds-cts-1 18652m 10827Mi ds-cts-2 19881m 11110Mi ds-idrepo-0 8000m 3893Mi ds-idrepo-1 7776m 3883Mi ds-idrepo-2 8111m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9332m 840Mi 20:20:58 DEBUG --- stderr --- 20:20:58 DEBUG 20:20:58 INFO [http_cmd]: http status code OK 20:20:58 DEBUG --- status code --- 20:20:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:58 DEBUG --- http response --- 20:20:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:58 DEBUG ----- output ----- 20:20:58 DEBUG { 20:20:58 DEBUG "status": "success", 20:20:58 DEBUG "data": { 20:20:58 DEBUG "resultType": "vector", 20:20:58 DEBUG "result": [ 20:20:58 DEBUG { 20:20:58 DEBUG "metric": { 20:20:58 DEBUG "pod": "ds-idrepo-0" 20:20:58 DEBUG }, 20:20:58 DEBUG "value": [ 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG [...] 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG "pod": "ds-idrepo-2" 20:20:58 DEBUG }, 20:20:58 DEBUG "value": [ 20:20:58 DEBUG 1713640858, 20:20:58 DEBUG "0" 20:20:58 DEBUG ] 20:20:58 DEBUG } 20:20:58 DEBUG ] 20:20:58 DEBUG } 20:20:58 DEBUG } 20:20:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640858" 20:20:58 INFO [http_cmd]: http status code OK 20:20:58 DEBUG --- status code --- 20:20:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:58 DEBUG --- http response --- 20:20:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:20:58 DEBUG ----- output ----- 20:20:58 DEBUG { 20:20:58 DEBUG "status": "success", 20:20:58 DEBUG "data": { 20:20:58 DEBUG "resultType": "vector", 20:20:58 DEBUG "result": [ 20:20:58 DEBUG { 20:20:58 DEBUG "metric": { 20:20:58 DEBUG "pod": "ds-cts-0" 20:20:58 DEBUG }, 20:20:58 DEBUG "value": [ 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG [...] 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG "pod": "ds-cts-2" 20:20:58 DEBUG }, 20:20:58 DEBUG "value": [ 20:20:58 DEBUG 1713640858, 20:20:58 DEBUG "0" 20:20:58 DEBUG ] 20:20:58 DEBUG } 20:20:58 DEBUG ] 20:20:58 DEBUG } 20:20:58 DEBUG } 20:20:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640858" 20:20:58 INFO [http_cmd]: http status code OK 20:20:58 DEBUG --- status code --- 20:20:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:58 DEBUG --- http response --- 20:20:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:20:58 DEBUG ----- output ----- 20:20:58 DEBUG { 20:20:58 DEBUG "status": "success", 20:20:58 DEBUG "data": { 20:20:58 DEBUG "resultType": "vector", 20:20:58 DEBUG "result": [ 20:20:58 DEBUG { 20:20:58 DEBUG "metric": { 20:20:58 DEBUG "pod": "ds-idrepo-0" 20:20:58 DEBUG }, 20:20:58 DEBUG "value": [ 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG [...] 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG "pod": "ds-cts-2" 20:20:58 DEBUG }, 20:20:58 DEBUG "value": [ 20:20:58 DEBUG 1713640858, 20:20:58 DEBUG "0" 20:20:58 DEBUG ] 20:20:58 DEBUG } 20:20:58 DEBUG ] 20:20:58 DEBUG } 20:20:58 DEBUG } 20:20:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640858" 20:20:58 INFO [http_cmd]: http status code OK 20:20:58 DEBUG --- status code --- 20:20:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:58 DEBUG --- http response --- 20:20:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:58 DEBUG ----- output ----- 20:20:58 DEBUG { 20:20:58 DEBUG "status": "success", 20:20:58 DEBUG "data": { 20:20:58 DEBUG "resultType": "vector", 20:20:58 DEBUG "result": [ 20:20:58 DEBUG { 20:20:58 DEBUG "metric": { 20:20:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:58 DEBUG }, 20:20:58 DEBUG "value": [ 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG [...] 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG 20:20:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:58 DEBUG }, 20:20:58 DEBUG "value": [ 20:20:58 DEBUG 1713640858, 20:20:58 DEBUG "510245" 20:20:58 DEBUG ] 20:20:58 DEBUG } 20:20:58 DEBUG ] 20:20:58 DEBUG } 20:20:58 DEBUG } 20:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640859" 20:20:59 INFO [http_cmd]: http status code OK 20:20:59 DEBUG --- status code --- 20:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:59 DEBUG --- http response --- 20:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:59 DEBUG ----- output ----- 20:20:59 DEBUG { 20:20:59 DEBUG "status": "success", 20:20:59 DEBUG "data": { 20:20:59 DEBUG "resultType": "vector", 20:20:59 DEBUG "result": [ 20:20:59 DEBUG { 20:20:59 DEBUG "metric": { 20:20:59 DEBUG "instance": "10.142.0.107:9100" 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG [...] 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG "instance": "10.142.0.43:9100" 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 1713640859, 20:20:59 DEBUG "0.18726666666665134" 20:20:59 DEBUG ] 20:20:59 DEBUG } 20:20:59 DEBUG ] 20:20:59 DEBUG } 20:20:59 DEBUG } 20:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640859" 20:20:59 INFO [http_cmd]: http status code OK 20:20:59 DEBUG --- status code --- 20:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:59 DEBUG --- http response --- 20:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:59 DEBUG ----- output ----- 20:20:59 DEBUG { 20:20:59 DEBUG "status": "success", 20:20:59 DEBUG "data": { 20:20:59 DEBUG "resultType": "vector", 20:20:59 DEBUG "result": [ 20:20:59 DEBUG { 20:20:59 DEBUG "metric": { 20:20:59 DEBUG "instance": "10.142.0.107:9100" 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG [...] 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG "instance": "10.142.0.43:9100" 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 1713640859, 20:20:59 DEBUG "85850534.19999999" 20:20:59 DEBUG ] 20:20:59 DEBUG } 20:20:59 DEBUG ] 20:20:59 DEBUG } 20:20:59 DEBUG } 20:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640859" 20:20:59 INFO 20:20:59 INFO [loop_until]: kubectl --namespace=xlou top node 20:20:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640859" 20:20:59 INFO [http_cmd]: http status code OK 20:20:59 DEBUG --- status code --- 20:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:59 DEBUG --- http response --- 20:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:59 DEBUG ----- output ----- 20:20:59 DEBUG { 20:20:59 DEBUG "status": "success", 20:20:59 DEBUG "data": { 20:20:59 DEBUG "resultType": "vector", 20:20:59 DEBUG "result": [ 20:20:59 DEBUG { 20:20:59 DEBUG "metric": { 20:20:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG [...] 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 1713640859, 20:20:59 DEBUG "979562496" 20:20:59 DEBUG ] 20:20:59 DEBUG } 20:20:59 DEBUG ] 20:20:59 DEBUG } 20:20:59 DEBUG } 20:20:59 INFO [http_cmd]: http status code OK 20:20:59 DEBUG --- status code --- 20:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:59 DEBUG --- http response --- 20:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:59 DEBUG ----- output ----- 20:20:59 DEBUG { 20:20:59 DEBUG "status": "success", 20:20:59 DEBUG "data": { 20:20:59 DEBUG "resultType": "vector", 20:20:59 DEBUG "result": [ 20:20:59 DEBUG { 20:20:59 DEBUG "metric": { 20:20:59 DEBUG "instance": "10.142.0.107:9100" 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG [...] 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG "instance": "10.142.0.43:9100" 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 1713640859, 20:20:59 DEBUG "85852303.83333333" 20:20:59 DEBUG ] 20:20:59 DEBUG } 20:20:59 DEBUG ] 20:20:59 DEBUG } 20:20:59 DEBUG } 20:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640859" 20:20:59 INFO [http_cmd]: http status code OK 20:20:59 DEBUG --- status code --- 20:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:59 DEBUG --- http response --- 20:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:59 DEBUG ----- output ----- 20:20:59 DEBUG { 20:20:59 DEBUG "status": "success", 20:20:59 DEBUG "data": { 20:20:59 DEBUG "resultType": "vector", 20:20:59 DEBUG "result": [ 20:20:59 DEBUG { 20:20:59 DEBUG "metric": { 20:20:59 DEBUG "pod": "ds-idrepo-1" 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG [...] 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG "pod": "ds-cts-0" 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 1713640859, 20:20:59 DEBUG "305275416.76636106" 20:20:59 DEBUG ] 20:20:59 DEBUG } 20:20:59 DEBUG ] 20:20:59 DEBUG } 20:20:59 DEBUG } 20:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640859" 20:20:59 INFO [http_cmd]: http status code OK 20:20:59 DEBUG --- status code --- 20:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:59 DEBUG --- http response --- 20:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:59 DEBUG ----- output ----- 20:20:59 DEBUG { 20:20:59 DEBUG "status": "success", 20:20:59 DEBUG "data": { 20:20:59 DEBUG "resultType": "vector", 20:20:59 DEBUG "result": [ 20:20:59 DEBUG { 20:20:59 DEBUG "metric": { 20:20:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG [...] 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 1713640859, 20:20:59 DEBUG "0" 20:20:59 DEBUG ] 20:20:59 DEBUG } 20:20:59 DEBUG ] 20:20:59 DEBUG } 20:20:59 DEBUG } 20:20:59 INFO [loop_until]: OK (rc = 0) 20:20:59 DEBUG --- stdout --- 20:20:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6213Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8188m 27% 7630Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18236m 61% 11387Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3341Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19569m 65% 14245Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 137m 0% 5207Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18608m 62% 11952Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15791m 52% 2963Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20906m 70% 10888Mi 9% gke-xlou-cdm-ds-0f601751-j488 8762m 29% 7239Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18690m 62% 15893Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8679m 29% 7847Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20155m 67% 13758Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9844m 61% 6397Mi 10% 20:20:59 DEBUG --- stderr --- 20:20:59 DEBUG 20:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640859" 20:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640859" 20:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640859" 20:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640859" 20:20:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640859" 20:20:59 INFO [http_cmd]: http status code OK 20:20:59 INFO [http_cmd]: http status code OK 20:20:59 DEBUG --- status code --- 20:20:59 INFO [http_cmd]: http status code OK 20:20:59 DEBUG --- status code --- 20:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:59 DEBUG --- status code --- 20:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:59 DEBUG --- http response --- 20:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:59 DEBUG --- http response --- 20:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:20:59 DEBUG --- http response --- 20:20:59 DEBUG ----- output ----- 20:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:20:59 DEBUG { 20:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:59 DEBUG ----- output ----- 20:20:59 DEBUG "status": "success", 20:20:59 DEBUG ----- output ----- 20:20:59 DEBUG { 20:20:59 INFO [http_cmd]: http status code OK 20:20:59 DEBUG "data": { 20:20:59 DEBUG { 20:20:59 DEBUG "status": "success", 20:20:59 DEBUG --- status code --- 20:20:59 DEBUG "resultType": "vector", 20:20:59 DEBUG "data": { 20:20:59 DEBUG "status": "success", 20:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:59 DEBUG "result": [ 20:20:59 DEBUG "resultType": "vector", 20:20:59 DEBUG "data": { 20:20:59 DEBUG --- http response --- 20:20:59 DEBUG { 20:20:59 DEBUG "result": [ 20:20:59 DEBUG { 20:20:59 DEBUG "resultType": "vector", 20:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:20:59 DEBUG ----- output ----- 20:20:59 DEBUG "metric": { 20:20:59 INFO [http_cmd]: http status code OK 20:20:59 DEBUG "result": [ 20:20:59 DEBUG { 20:20:59 DEBUG "metric": { 20:20:59 DEBUG "pod": "ds-idrepo-1" 20:20:59 DEBUG --- status code --- 20:20:59 DEBUG { 20:20:59 DEBUG "status": "success", 20:20:59 DEBUG "pod": "ds-cts-0" 20:20:59 DEBUG }, 20:20:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:20:59 DEBUG "metric": { 20:20:59 DEBUG "data": { 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG --- http response --- 20:20:59 DEBUG "pod": "ds-cts-0" 20:20:59 DEBUG "resultType": "vector", 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG [...] 20:20:59 DEBUG 20:20:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:20:59 DEBUG 20:20:59 DEBUG "result": [ 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG }, 20:20:59 DEBUG 20:20:59 DEBUG { 20:20:59 DEBUG [...] 20:20:59 DEBUG ----- output ----- 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 20:20:59 DEBUG "metric": { 20:20:59 DEBUG 20:20:59 DEBUG { 20:20:59 DEBUG 20:20:59 DEBUG 20:20:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:20:59 DEBUG 20:20:59 DEBUG "status": "success", 20:20:59 DEBUG 20:20:59 DEBUG "pod": "ds-cts-0" 20:20:59 DEBUG }, 20:20:59 DEBUG 20:20:59 DEBUG "data": { 20:20:59 DEBUG [...] 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG "pod": "ds-cts-2" 20:20:59 DEBUG "resultType": "vector", 20:20:59 DEBUG 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 20:20:59 DEBUG }, 20:20:59 DEBUG "result": [ 20:20:59 DEBUG 20:20:59 DEBUG 1713640859, 20:20:59 DEBUG 20:20:59 DEBUG "value": [ 20:20:59 DEBUG { 20:20:59 DEBUG 20:20:59 DEBUG "17636225.991920467" 20:20:59 DEBUG 20:20:59 DEBUG 1713640859, 20:20:59 DEBUG "metric": { 20:20:59 DEBUG "pod": "ds-cts-2" 20:20:59 DEBUG ] 20:20:59 DEBUG [...] 20:20:59 DEBUG "31.7" 20:20:59 DEBUG "instance": "10.142.0.107:9100" 20:20:59 DEBUG }, 20:20:59 DEBUG } 20:20:59 DEBUG 20:20:59 DEBUG ] 20:20:59 DEBUG }, 20:20:59 DEBUG "value": [ 20:20:59 DEBUG ] 20:20:59 DEBUG 20:20:59 DEBUG } 20:20:59 DEBUG "value": [ 20:20:59 DEBUG 1713640859, 20:20:59 DEBUG } 20:20:59 DEBUG 20:21:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:20:59 DEBUG 20:20:59 DEBUG "0.001566666666666668" 20:20:59 DEBUG } 20:21:00 DEBUG }, 20:20:59 DEBUG ] 20:21:00 DEBUG 20:21:00 DEBUG ] 20:21:00 DEBUG "value": [ 20:21:00 DEBUG } 20:21:00 DEBUG 20:21:00 DEBUG [...] 20:21:00 DEBUG 1713640859, 20:21:00 DEBUG } 20:21:00 DEBUG } 20:21:00 DEBUG ] 20:21:00 DEBUG "0.159" 20:21:00 DEBUG } 20:21:00 DEBUG 20:21:00 DEBUG 20:21:00 DEBUG } 20:21:00 DEBUG ] 20:21:00 DEBUG 20:21:00 DEBUG } 20:21:00 DEBUG "instance": "10.142.0.43:9100" 20:21:00 DEBUG ] 20:21:00 DEBUG }, 20:21:00 DEBUG } 20:21:00 DEBUG "value": [ 20:21:00 DEBUG } 20:21:00 DEBUG 1713640859, 20:21:00 DEBUG "0.06966666666666015" 20:21:00 DEBUG ] 20:21:00 DEBUG } 20:21:00 DEBUG ] 20:21:00 DEBUG } 20:21:00 DEBUG } 20:21:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640914" 20:21:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640914" 20:21:54 ERROR [http_cmd]: ERROR 20:21:54 INFO [http_cmd]: http status code OK 20:21:54 ERROR --- status code --- 20:21:54 DEBUG --- status code --- 20:21:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:21:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:54 ERROR --- http response --- 20:21:54 DEBUG --- http response --- 20:21:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:21:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:21:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:21:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640915" 20:21:55 INFO [http_cmd]: http status code OK 20:21:55 DEBUG --- status code --- 20:21:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:55 DEBUG --- http response --- 20:21:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:55 DEBUG ----- output ----- 20:21:55 DEBUG { 20:21:55 DEBUG "status": "success", 20:21:55 DEBUG "data": { 20:21:55 DEBUG "resultType": "vector", 20:21:55 DEBUG "result": [ 20:21:55 DEBUG { 20:21:55 DEBUG "metric": { 20:21:55 DEBUG "pod": "ds-idrepo-0" 20:21:55 DEBUG }, 20:21:55 DEBUG "value": [ 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG [...] 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG "pod": "ds-idrepo-2" 20:21:55 DEBUG }, 20:21:55 DEBUG "value": [ 20:21:55 DEBUG 1713640915, 20:21:55 DEBUG "0" 20:21:55 DEBUG ] 20:21:55 DEBUG } 20:21:55 DEBUG ] 20:21:55 DEBUG } 20:21:55 DEBUG } 20:21:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640915" 20:21:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640915" 20:21:55 INFO [http_cmd]: http status code OK 20:21:55 INFO [http_cmd]: http status code OK 20:21:55 DEBUG --- status code --- 20:21:55 DEBUG --- status code --- 20:21:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:55 DEBUG --- http response --- 20:21:55 DEBUG --- http response --- 20:21:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:55 DEBUG ----- output ----- 20:21:55 DEBUG ----- output ----- 20:21:55 DEBUG { 20:21:55 DEBUG { 20:21:55 DEBUG "status": "success", 20:21:55 DEBUG "status": "success", 20:21:55 DEBUG "data": { 20:21:55 DEBUG "data": { 20:21:55 DEBUG "resultType": "vector", 20:21:55 DEBUG "resultType": "vector", 20:21:55 DEBUG "result": [ 20:21:55 DEBUG "result": [ 20:21:55 DEBUG { 20:21:55 DEBUG { 20:21:55 DEBUG "metric": { 20:21:55 DEBUG "metric": { 20:21:55 DEBUG "pod": "ds-idrepo-0" 20:21:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:55 DEBUG }, 20:21:55 DEBUG }, 20:21:55 DEBUG "value": [ 20:21:55 DEBUG "value": [ 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG [...] 20:21:55 DEBUG [...] 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG 20:21:55 DEBUG "pod": "ds-idrepo-2" 20:21:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:55 DEBUG }, 20:21:55 DEBUG }, 20:21:55 DEBUG "value": [ 20:21:55 DEBUG "value": [ 20:21:55 DEBUG 1713640915, 20:21:55 DEBUG 1713640915, 20:21:55 DEBUG "0" 20:21:55 DEBUG "2827.1" 20:21:55 DEBUG ] 20:21:55 DEBUG ] 20:21:55 DEBUG } 20:21:55 DEBUG } 20:21:55 DEBUG ] 20:21:55 DEBUG ] 20:21:55 DEBUG } 20:21:55 DEBUG } 20:21:55 DEBUG } 20:21:55 DEBUG } 20:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640916" 20:21:56 INFO [http_cmd]: http status code OK 20:21:56 DEBUG --- status code --- 20:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:56 DEBUG --- http response --- 20:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:21:56 DEBUG ----- output ----- 20:21:56 DEBUG { 20:21:56 DEBUG "status": "success", 20:21:56 DEBUG "data": { 20:21:56 DEBUG "resultType": "vector", 20:21:56 DEBUG "result": [ 20:21:56 DEBUG { 20:21:56 DEBUG "metric": { 20:21:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:21:56 DEBUG }, 20:21:56 DEBUG "value": [ 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG [...] 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:21:56 DEBUG }, 20:21:56 DEBUG "value": [ 20:21:56 DEBUG 1713640916, 20:21:56 DEBUG "1076.5902715110838" 20:21:56 DEBUG ] 20:21:56 DEBUG } 20:21:56 DEBUG ] 20:21:56 DEBUG } 20:21:56 DEBUG } 20:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640916" 20:21:56 INFO [http_cmd]: http status code OK 20:21:56 DEBUG --- status code --- 20:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:56 DEBUG --- http response --- 20:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:56 DEBUG ----- output ----- 20:21:56 DEBUG { 20:21:56 DEBUG "status": "success", 20:21:56 DEBUG "data": { 20:21:56 DEBUG "resultType": "vector", 20:21:56 DEBUG "result": [ 20:21:56 DEBUG { 20:21:56 DEBUG "metric": { 20:21:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:56 DEBUG }, 20:21:56 DEBUG "value": [ 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG [...] 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:56 DEBUG }, 20:21:56 DEBUG "value": [ 20:21:56 DEBUG 1713640916, 20:21:56 DEBUG "2840.9333333333334" 20:21:56 DEBUG ] 20:21:56 DEBUG } 20:21:56 DEBUG ] 20:21:56 DEBUG } 20:21:56 DEBUG } 20:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640916" 20:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640916" 20:21:56 INFO [http_cmd]: http status code OK 20:21:56 DEBUG --- status code --- 20:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:56 DEBUG --- http response --- 20:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:56 DEBUG ----- output ----- 20:21:56 DEBUG { 20:21:56 DEBUG "status": "success", 20:21:56 DEBUG "data": { 20:21:56 DEBUG "resultType": "vector", 20:21:56 DEBUG "result": [ 20:21:56 DEBUG { 20:21:56 DEBUG "metric": { 20:21:56 DEBUG "instance": "10.142.0.107:9100" 20:21:56 DEBUG }, 20:21:56 DEBUG "value": [ 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG [...] 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG "instance": "10.142.0.43:9100" 20:21:56 DEBUG }, 20:21:56 DEBUG "value": [ 20:21:56 DEBUG 1713640916, 20:21:56 DEBUG "3737809486848" 20:21:56 DEBUG ] 20:21:56 DEBUG } 20:21:56 DEBUG ] 20:21:56 DEBUG } 20:21:56 DEBUG } 20:21:56 INFO [http_cmd]: http status code OK 20:21:56 DEBUG --- status code --- 20:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:56 DEBUG --- http response --- 20:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:56 DEBUG ----- output ----- 20:21:56 DEBUG { 20:21:56 DEBUG "status": "success", 20:21:56 DEBUG "data": { 20:21:56 DEBUG "resultType": "vector", 20:21:56 DEBUG "result": [ 20:21:56 DEBUG { 20:21:56 DEBUG "metric": { 20:21:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:21:56 DEBUG }, 20:21:56 DEBUG "value": [ 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG [...] 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG 20:21:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:21:56 DEBUG }, 20:21:56 DEBUG "value": [ 20:21:56 DEBUG 1713640916, 20:21:56 DEBUG "9.604830749127009" 20:21:56 DEBUG ] 20:21:56 DEBUG } 20:21:56 DEBUG ] 20:21:56 DEBUG } 20:21:56 DEBUG } 20:21:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640917" 20:21:57 INFO [http_cmd]: http status code OK 20:21:57 DEBUG --- status code --- 20:21:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:57 DEBUG --- http response --- 20:21:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:57 DEBUG ----- output ----- 20:21:57 DEBUG { 20:21:57 DEBUG "status": "success", 20:21:57 DEBUG "data": { 20:21:57 DEBUG "resultType": "vector", 20:21:57 DEBUG "result": [ 20:21:57 DEBUG { 20:21:57 DEBUG "metric": { 20:21:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:57 DEBUG }, 20:21:57 DEBUG "value": [ 20:21:57 DEBUG 20:21:57 DEBUG 20:21:57 DEBUG 20:21:57 DEBUG [...] 20:21:57 DEBUG 20:21:57 DEBUG 20:21:57 DEBUG 20:21:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:57 DEBUG }, 20:21:57 DEBUG "value": [ 20:21:57 DEBUG 1713640917, 20:21:57 DEBUG "2842.6666666666665" 20:21:57 DEBUG ] 20:21:57 DEBUG } 20:21:57 DEBUG ] 20:21:57 DEBUG } 20:21:57 DEBUG } 20:21:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640917" 20:21:57 INFO [http_cmd]: http status code OK 20:21:57 DEBUG --- status code --- 20:21:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:57 DEBUG --- http response --- 20:21:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:57 DEBUG ----- output ----- 20:21:57 DEBUG { 20:21:57 DEBUG "status": "success", 20:21:57 DEBUG "data": { 20:21:57 DEBUG "resultType": "vector", 20:21:57 DEBUG "result": [ 20:21:57 DEBUG { 20:21:57 DEBUG "metric": { 20:21:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:57 DEBUG }, 20:21:57 DEBUG "value": [ 20:21:57 DEBUG 20:21:57 DEBUG 20:21:57 DEBUG 20:21:57 DEBUG [...] 20:21:57 DEBUG 20:21:57 DEBUG 20:21:57 DEBUG 20:21:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:57 DEBUG }, 20:21:57 DEBUG "value": [ 20:21:57 DEBUG 1713640917, 20:21:57 DEBUG "2844.3" 20:21:57 DEBUG ] 20:21:57 DEBUG } 20:21:57 DEBUG ] 20:21:57 DEBUG } 20:21:57 DEBUG } 20:21:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640917" 20:21:57 INFO [http_cmd]: http status code OK 20:21:57 DEBUG --- status code --- 20:21:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:57 DEBUG --- http response --- 20:21:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:57 DEBUG ----- output ----- 20:21:57 DEBUG { 20:21:57 DEBUG "status": "success", 20:21:57 DEBUG "data": { 20:21:57 DEBUG "resultType": "vector", 20:21:57 DEBUG "result": [ 20:21:57 DEBUG { 20:21:57 DEBUG "metric": { 20:21:57 DEBUG "pod": "ds-idrepo-1" 20:21:57 DEBUG }, 20:21:57 DEBUG "value": [ 20:21:57 DEBUG 20:21:57 DEBUG 20:21:57 DEBUG 20:21:57 DEBUG [...] 20:21:57 DEBUG 20:21:57 DEBUG 20:21:57 DEBUG 20:21:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:21:57 DEBUG }, 20:21:57 DEBUG "value": [ 20:21:57 DEBUG 1713640917, 20:21:57 DEBUG "0.000029391115649730332" 20:21:57 DEBUG ] 20:21:57 DEBUG } 20:21:57 DEBUG ] 20:21:57 DEBUG } 20:21:57 DEBUG } 20:21:58 INFO 20:21:58 INFO [loop_until]: kubectl --namespace=xlou top pods 20:21:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:21:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640918" 20:21:58 INFO [http_cmd]: http status code OK 20:21:58 DEBUG --- status code --- 20:21:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:58 DEBUG --- http response --- 20:21:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:58 DEBUG ----- output ----- 20:21:58 DEBUG { 20:21:58 DEBUG "status": "success", 20:21:58 DEBUG "data": { 20:21:58 DEBUG "resultType": "vector", 20:21:58 DEBUG "result": [ 20:21:58 DEBUG { 20:21:58 DEBUG "metric": { 20:21:58 DEBUG "pod": "ds-idrepo-0" 20:21:58 DEBUG }, 20:21:58 DEBUG "value": [ 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG [...] 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG "pod": "ds-idrepo-2" 20:21:58 DEBUG }, 20:21:58 DEBUG "value": [ 20:21:58 DEBUG 1713640918, 20:21:58 DEBUG "0" 20:21:58 DEBUG ] 20:21:58 DEBUG } 20:21:58 DEBUG ] 20:21:58 DEBUG } 20:21:58 DEBUG } 20:21:58 INFO [loop_until]: OK (rc = 0) 20:21:58 DEBUG --- stdout --- 20:21:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18426m 8661Mi am-767f6b67b5-qcqz2 18572m 9071Mi am-767f6b67b5-vbltz 19916m 8671Mi ds-cts-0 19193m 11037Mi ds-cts-1 18750m 10847Mi ds-cts-2 20276m 11104Mi ds-idrepo-0 7780m 3893Mi ds-idrepo-1 7913m 3882Mi ds-idrepo-2 7882m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9589m 842Mi 20:21:58 DEBUG --- stderr --- 20:21:58 DEBUG 20:21:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640918" 20:21:58 INFO [http_cmd]: http status code OK 20:21:58 DEBUG --- status code --- 20:21:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:58 DEBUG --- http response --- 20:21:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:21:58 DEBUG ----- output ----- 20:21:58 DEBUG { 20:21:58 DEBUG "status": "success", 20:21:58 DEBUG "data": { 20:21:58 DEBUG "resultType": "vector", 20:21:58 DEBUG "result": [ 20:21:58 DEBUG { 20:21:58 DEBUG "metric": { 20:21:58 DEBUG "pod": "ds-cts-0" 20:21:58 DEBUG }, 20:21:58 DEBUG "value": [ 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG [...] 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG "pod": "ds-cts-2" 20:21:58 DEBUG }, 20:21:58 DEBUG "value": [ 20:21:58 DEBUG 1713640918, 20:21:58 DEBUG "0" 20:21:58 DEBUG ] 20:21:58 DEBUG } 20:21:58 DEBUG ] 20:21:58 DEBUG } 20:21:58 DEBUG } 20:21:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640918" 20:21:58 INFO [http_cmd]: http status code OK 20:21:58 DEBUG --- status code --- 20:21:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:58 DEBUG --- http response --- 20:21:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:21:58 DEBUG ----- output ----- 20:21:58 DEBUG { 20:21:58 DEBUG "status": "success", 20:21:58 DEBUG "data": { 20:21:58 DEBUG "resultType": "vector", 20:21:58 DEBUG "result": [ 20:21:58 DEBUG { 20:21:58 DEBUG "metric": { 20:21:58 DEBUG "pod": "ds-idrepo-0" 20:21:58 DEBUG }, 20:21:58 DEBUG "value": [ 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG [...] 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG "pod": "ds-cts-2" 20:21:58 DEBUG }, 20:21:58 DEBUG "value": [ 20:21:58 DEBUG 1713640918, 20:21:58 DEBUG "0" 20:21:58 DEBUG ] 20:21:58 DEBUG } 20:21:58 DEBUG ] 20:21:58 DEBUG } 20:21:58 DEBUG } 20:21:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640918" 20:21:58 INFO [http_cmd]: http status code OK 20:21:58 DEBUG --- status code --- 20:21:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:58 DEBUG --- http response --- 20:21:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:58 DEBUG ----- output ----- 20:21:58 DEBUG { 20:21:58 DEBUG "status": "success", 20:21:58 DEBUG "data": { 20:21:58 DEBUG "resultType": "vector", 20:21:58 DEBUG "result": [ 20:21:58 DEBUG { 20:21:58 DEBUG "metric": { 20:21:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:58 DEBUG }, 20:21:58 DEBUG "value": [ 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG [...] 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG 20:21:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:58 DEBUG }, 20:21:58 DEBUG "value": [ 20:21:58 DEBUG 1713640918, 20:21:58 DEBUG "510403" 20:21:58 DEBUG ] 20:21:58 DEBUG } 20:21:58 DEBUG ] 20:21:58 DEBUG } 20:21:58 DEBUG } 20:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640919" 20:21:59 INFO [http_cmd]: http status code OK 20:21:59 DEBUG --- status code --- 20:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:59 DEBUG --- http response --- 20:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:59 DEBUG ----- output ----- 20:21:59 DEBUG { 20:21:59 DEBUG "status": "success", 20:21:59 DEBUG "data": { 20:21:59 DEBUG "resultType": "vector", 20:21:59 DEBUG "result": [ 20:21:59 DEBUG { 20:21:59 DEBUG "metric": { 20:21:59 DEBUG "instance": "10.142.0.107:9100" 20:21:59 DEBUG }, 20:21:59 DEBUG "value": [ 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG [...] 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG "instance": "10.142.0.43:9100" 20:21:59 DEBUG }, 20:21:59 DEBUG "value": [ 20:21:59 DEBUG 1713640919, 20:21:59 DEBUG "0.2659666666667249" 20:21:59 DEBUG ] 20:21:59 DEBUG } 20:21:59 DEBUG ] 20:21:59 DEBUG } 20:21:59 DEBUG } 20:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640919" 20:21:59 INFO [http_cmd]: http status code OK 20:21:59 DEBUG --- status code --- 20:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:59 DEBUG --- http response --- 20:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:59 DEBUG ----- output ----- 20:21:59 DEBUG { 20:21:59 DEBUG "status": "success", 20:21:59 DEBUG "data": { 20:21:59 DEBUG "resultType": "vector", 20:21:59 DEBUG "result": [ 20:21:59 DEBUG { 20:21:59 DEBUG "metric": { 20:21:59 DEBUG "instance": "10.142.0.107:9100" 20:21:59 DEBUG }, 20:21:59 DEBUG "value": [ 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG [...] 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG "instance": "10.142.0.43:9100" 20:21:59 DEBUG }, 20:21:59 DEBUG "value": [ 20:21:59 DEBUG 1713640919, 20:21:59 DEBUG "85859900.8" 20:21:59 DEBUG ] 20:21:59 DEBUG } 20:21:59 DEBUG ] 20:21:59 DEBUG } 20:21:59 DEBUG } 20:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640919" 20:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640919" 20:21:59 INFO [http_cmd]: http status code OK 20:21:59 DEBUG --- status code --- 20:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:59 DEBUG --- http response --- 20:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:59 DEBUG ----- output ----- 20:21:59 DEBUG { 20:21:59 DEBUG "status": "success", 20:21:59 DEBUG "data": { 20:21:59 DEBUG "resultType": "vector", 20:21:59 DEBUG "result": [ 20:21:59 DEBUG { 20:21:59 DEBUG "metric": { 20:21:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:21:59 DEBUG }, 20:21:59 DEBUG "value": [ 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG [...] 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:21:59 DEBUG }, 20:21:59 DEBUG "value": [ 20:21:59 DEBUG 1713640919, 20:21:59 DEBUG "981712896" 20:21:59 DEBUG ] 20:21:59 DEBUG } 20:21:59 DEBUG ] 20:21:59 DEBUG } 20:21:59 DEBUG } 20:21:59 INFO [http_cmd]: http status code OK 20:21:59 DEBUG --- status code --- 20:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:59 DEBUG --- http response --- 20:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:59 DEBUG ----- output ----- 20:21:59 DEBUG { 20:21:59 DEBUG "status": "success", 20:21:59 DEBUG "data": { 20:21:59 DEBUG "resultType": "vector", 20:21:59 DEBUG "result": [ 20:21:59 DEBUG { 20:21:59 DEBUG "metric": { 20:21:59 DEBUG "instance": "10.142.0.107:9100" 20:21:59 DEBUG }, 20:21:59 DEBUG "value": [ 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG [...] 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG "instance": "10.142.0.43:9100" 20:21:59 DEBUG }, 20:21:59 DEBUG "value": [ 20:21:59 DEBUG 1713640919, 20:21:59 DEBUG "85861907.80000001" 20:21:59 DEBUG ] 20:21:59 DEBUG } 20:21:59 DEBUG ] 20:21:59 DEBUG } 20:21:59 DEBUG } 20:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640919" 20:21:59 INFO [http_cmd]: http status code OK 20:21:59 DEBUG --- status code --- 20:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:59 DEBUG --- http response --- 20:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:21:59 DEBUG ----- output ----- 20:21:59 DEBUG { 20:21:59 DEBUG "status": "success", 20:21:59 DEBUG "data": { 20:21:59 DEBUG "resultType": "vector", 20:21:59 DEBUG "result": [ 20:21:59 DEBUG { 20:21:59 DEBUG "metric": { 20:21:59 DEBUG "pod": "ds-idrepo-1" 20:21:59 DEBUG }, 20:21:59 DEBUG "value": [ 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG [...] 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:21:59 DEBUG }, 20:21:59 DEBUG "value": [ 20:21:59 DEBUG 1713640919, 20:21:59 DEBUG "0" 20:21:59 DEBUG ] 20:21:59 DEBUG } 20:21:59 DEBUG ] 20:21:59 DEBUG } 20:21:59 DEBUG } 20:21:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640919" 20:21:59 INFO [http_cmd]: http status code OK 20:21:59 DEBUG --- status code --- 20:21:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:21:59 DEBUG --- http response --- 20:21:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:21:59 DEBUG ----- output ----- 20:21:59 DEBUG { 20:21:59 DEBUG "status": "success", 20:21:59 DEBUG "data": { 20:21:59 DEBUG "resultType": "vector", 20:21:59 DEBUG "result": [ 20:21:59 DEBUG { 20:21:59 DEBUG "metric": { 20:21:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:21:59 DEBUG }, 20:21:59 DEBUG "value": [ 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG [...] 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG 20:21:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:21:59 DEBUG }, 20:21:59 DEBUG "value": [ 20:21:59 DEBUG 1713640919, 20:21:59 DEBUG "0" 20:21:59 DEBUG ] 20:21:59 DEBUG } 20:21:59 DEBUG ] 20:21:59 DEBUG } 20:21:59 DEBUG } 20:21:59 INFO 20:21:59 INFO [loop_until]: kubectl --namespace=xlou top node 20:21:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:22:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640920" 20:22:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640920" 20:22:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640920" 20:22:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640920" 20:22:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640920" 20:22:00 INFO [loop_until]: OK (rc = 0) 20:22:00 DEBUG --- stdout --- 20:22:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6217Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8188m 27% 7636Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17649m 59% 11386Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3339Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19449m 65% 14205Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5210Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18549m 62% 11944Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15777m 52% 2977Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20741m 69% 10884Mi 9% gke-xlou-cdm-ds-0f601751-j488 8785m 29% 7228Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18222m 61% 15884Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8653m 28% 7850Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20174m 67% 13755Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9801m 61% 6398Mi 10% 20:22:00 DEBUG --- stderr --- 20:22:00 DEBUG 20:22:00 INFO [http_cmd]: http status code OK 20:22:00 INFO [http_cmd]: http status code OK 20:22:00 DEBUG --- status code --- 20:22:00 DEBUG --- status code --- 20:22:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:00 DEBUG --- http response --- 20:22:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:00 INFO [http_cmd]: http status code OK 20:22:00 DEBUG --- status code --- 20:22:00 DEBUG --- http response --- 20:22:00 INFO [http_cmd]: http status code OK 20:22:00 DEBUG --- status code --- 20:22:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:22:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:00 DEBUG ----- output ----- 20:22:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:22:00 DEBUG --- http response --- 20:22:00 DEBUG --- http response --- 20:22:00 DEBUG { 20:22:00 DEBUG ----- output ----- 20:22:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:22:00 DEBUG "status": "success", 20:22:00 DEBUG { 20:22:00 DEBUG ----- output ----- 20:22:00 DEBUG ----- output ----- 20:22:00 DEBUG "data": { 20:22:00 DEBUG "status": "success", 20:22:00 DEBUG { 20:22:00 DEBUG { 20:22:00 DEBUG "resultType": "vector", 20:22:00 DEBUG "data": { 20:22:00 DEBUG "status": "success", 20:22:00 INFO [http_cmd]: http status code OK 20:22:00 DEBUG "status": "success", 20:22:00 DEBUG "result": [ 20:22:00 DEBUG "resultType": "vector", 20:22:00 DEBUG "data": { 20:22:00 DEBUG --- status code --- 20:22:00 DEBUG "data": { 20:22:00 DEBUG { 20:22:00 DEBUG "metric": { 20:22:00 DEBUG "resultType": "vector", 20:22:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:00 DEBUG --- http response --- 20:22:00 DEBUG "result": [ 20:22:00 DEBUG "pod": "ds-idrepo-1" 20:22:00 DEBUG "result": [ 20:22:00 DEBUG "resultType": "vector", 20:22:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:00 DEBUG { 20:22:00 DEBUG }, 20:22:00 DEBUG { 20:22:00 DEBUG "result": [ 20:22:00 DEBUG ----- output ----- 20:22:00 DEBUG "metric": { 20:22:00 DEBUG "value": [ 20:22:00 DEBUG "metric": { 20:22:00 DEBUG { 20:22:00 DEBUG { 20:22:00 DEBUG "pod": "ds-cts-0" 20:22:00 DEBUG 20:22:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:00 DEBUG }, 20:22:00 DEBUG "status": "success", 20:22:00 DEBUG }, 20:22:00 DEBUG 20:22:00 DEBUG "value": [ 20:22:00 DEBUG "metric": { 20:22:00 DEBUG "data": { 20:22:00 DEBUG "value": [ 20:22:00 DEBUG 20:22:00 DEBUG 20:22:00 DEBUG "pod": "ds-cts-0" 20:22:00 DEBUG "resultType": "vector", 20:22:00 DEBUG "result": [ 20:22:00 DEBUG { 20:22:00 DEBUG 20:22:00 DEBUG }, 20:22:00 DEBUG 20:22:00 DEBUG "metric": { 20:22:00 DEBUG [...] 20:22:00 DEBUG 20:22:00 DEBUG [...] 20:22:00 DEBUG 20:22:00 DEBUG 20:22:00 DEBUG 20:22:00 DEBUG 20:22:00 DEBUG "value": [ 20:22:00 DEBUG [...] 20:22:00 DEBUG 20:22:00 DEBUG "instance": "10.142.0.107:9100" 20:22:00 DEBUG 20:22:00 DEBUG 20:22:00 DEBUG 20:22:00 DEBUG 20:22:00 DEBUG }, 20:22:00 DEBUG 20:22:00 DEBUG 20:22:00 DEBUG 20:22:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:22:00 DEBUG "value": [ 20:22:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:00 DEBUG }, 20:22:00 DEBUG "value": [ 20:22:00 DEBUG 1713640920, 20:22:00 DEBUG 20:22:00 DEBUG "0.159" 20:22:00 DEBUG 20:22:00 DEBUG }, 20:22:00 DEBUG 20:22:00 DEBUG ] 20:22:00 DEBUG 20:22:00 DEBUG [...] 20:22:00 DEBUG 20:22:00 DEBUG 20:22:00 DEBUG } 20:22:00 DEBUG "pod": "ds-cts-2" 20:22:00 DEBUG 20:22:00 DEBUG [...] 20:22:00 DEBUG ] 20:22:00 DEBUG }, 20:22:00 DEBUG "value": [ 20:22:00 DEBUG 20:22:00 DEBUG 20:22:00 DEBUG } 20:22:00 DEBUG 1713640920, 20:22:00 DEBUG "30.133333333333333" 20:22:00 DEBUG ] 20:22:00 DEBUG "value": [ 20:22:00 DEBUG } 20:22:00 DEBUG 20:22:00 DEBUG } 20:22:00 DEBUG ] 20:22:00 DEBUG 1713640920, 20:22:00 DEBUG 20:22:00 DEBUG } 20:22:00 DEBUG "pod": "ds-cts-2" 20:22:00 DEBUG "0" 20:22:00 DEBUG "instance": "10.142.0.43:9100" 20:22:00 DEBUG } 20:22:00 DEBUG }, 20:22:00 DEBUG ] 20:22:00 DEBUG } 20:22:00 DEBUG "value": [ 20:22:00 DEBUG ] 20:22:00 DEBUG }, 20:22:00 DEBUG 1713640920, 20:22:00 DEBUG } 20:22:00 DEBUG "value": [ 20:22:00 DEBUG "0.022302362222222226" 20:22:00 DEBUG } 20:22:00 DEBUG 1713640920, 20:22:00 DEBUG ] 20:22:00 DEBUG "0.09033333333333832" 20:22:00 DEBUG } 20:22:00 DEBUG ] 20:22:00 DEBUG ] 20:22:00 DEBUG } 20:22:00 DEBUG } 20:22:00 DEBUG ] 20:22:00 DEBUG } 20:22:00 DEBUG } 20:22:00 DEBUG } 20:22:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640974" 20:22:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713640974" 20:22:54 INFO [http_cmd]: http status code OK 20:22:54 DEBUG --- status code --- 20:22:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:54 DEBUG --- http response --- 20:22:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:22:54 ERROR [http_cmd]: ERROR 20:22:54 ERROR --- status code --- 20:22:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:22:54 ERROR --- http response --- 20:22:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:22:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:22:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640975" 20:22:55 INFO [http_cmd]: http status code OK 20:22:55 DEBUG --- status code --- 20:22:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:55 DEBUG --- http response --- 20:22:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:55 DEBUG ----- output ----- 20:22:55 DEBUG { 20:22:55 DEBUG "status": "success", 20:22:55 DEBUG "data": { 20:22:55 DEBUG "resultType": "vector", 20:22:55 DEBUG "result": [ 20:22:55 DEBUG { 20:22:55 DEBUG "metric": { 20:22:55 DEBUG "pod": "ds-idrepo-0" 20:22:55 DEBUG }, 20:22:55 DEBUG "value": [ 20:22:55 DEBUG 20:22:55 DEBUG 20:22:55 DEBUG 20:22:55 DEBUG [...] 20:22:55 DEBUG 20:22:55 DEBUG 20:22:55 DEBUG 20:22:55 DEBUG "pod": "ds-idrepo-2" 20:22:55 DEBUG }, 20:22:55 DEBUG "value": [ 20:22:55 DEBUG 1713640975, 20:22:55 DEBUG "0" 20:22:55 DEBUG ] 20:22:55 DEBUG } 20:22:55 DEBUG ] 20:22:55 DEBUG } 20:22:55 DEBUG } 20:22:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640975" 20:22:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640975" 20:22:55 INFO [http_cmd]: http status code OK 20:22:55 INFO [http_cmd]: http status code OK 20:22:55 DEBUG --- status code --- 20:22:55 DEBUG --- status code --- 20:22:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:55 DEBUG --- http response --- 20:22:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:55 DEBUG --- http response --- 20:22:55 DEBUG ----- output ----- 20:22:55 DEBUG { 20:22:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:55 DEBUG "status": "success", 20:22:55 DEBUG ----- output ----- 20:22:55 DEBUG "data": { 20:22:55 DEBUG { 20:22:55 DEBUG "resultType": "vector", 20:22:55 DEBUG "status": "success", 20:22:55 DEBUG "result": [ 20:22:55 DEBUG { 20:22:55 DEBUG "metric": { 20:22:55 DEBUG "pod": "ds-idrepo-0" 20:22:55 DEBUG }, 20:22:55 DEBUG "value": [ 20:22:55 DEBUG 20:22:55 DEBUG 20:22:55 DEBUG "data": { 20:22:55 DEBUG "resultType": "vector", 20:22:55 DEBUG 20:22:55 DEBUG "result": [ 20:22:55 DEBUG [...] 20:22:55 DEBUG { 20:22:55 DEBUG 20:22:55 DEBUG "metric": { 20:22:55 DEBUG 20:22:55 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:55 DEBUG 20:22:55 DEBUG }, 20:22:55 DEBUG "pod": "ds-idrepo-2" 20:22:55 DEBUG "value": [ 20:22:55 DEBUG }, 20:22:55 DEBUG 20:22:55 DEBUG "value": [ 20:22:55 DEBUG 20:22:55 DEBUG 1713640975, 20:22:55 DEBUG 20:22:55 DEBUG "0" 20:22:55 DEBUG [...] 20:22:55 DEBUG ] 20:22:55 DEBUG 20:22:55 DEBUG } 20:22:55 DEBUG 20:22:55 DEBUG ] 20:22:55 DEBUG 20:22:55 DEBUG } 20:22:55 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:55 DEBUG } 20:22:55 DEBUG }, 20:22:55 DEBUG "value": [ 20:22:55 DEBUG 1713640975, 20:22:55 DEBUG "2831.5" 20:22:55 DEBUG ] 20:22:55 DEBUG } 20:22:55 DEBUG ] 20:22:55 DEBUG } 20:22:55 DEBUG } 20:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640976" 20:22:56 INFO [http_cmd]: http status code OK 20:22:56 DEBUG --- status code --- 20:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:56 DEBUG --- http response --- 20:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:56 DEBUG ----- output ----- 20:22:56 DEBUG { 20:22:56 DEBUG "status": "success", 20:22:56 DEBUG "data": { 20:22:56 DEBUG "resultType": "vector", 20:22:56 DEBUG "result": [ 20:22:56 DEBUG { 20:22:56 DEBUG "metric": { 20:22:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:22:56 DEBUG }, 20:22:56 DEBUG "value": [ 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG [...] 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG "pod": "idm-b85f9d978-txpwv" 20:22:56 DEBUG }, 20:22:56 DEBUG "value": [ 20:22:56 DEBUG 1713640976, 20:22:56 DEBUG "478.15555231285566" 20:22:56 DEBUG ] 20:22:56 DEBUG } 20:22:56 DEBUG ] 20:22:56 DEBUG } 20:22:56 DEBUG } 20:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640976" 20:22:56 INFO [http_cmd]: http status code OK 20:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640976" 20:22:56 DEBUG --- status code --- 20:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713640976" 20:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:56 DEBUG --- http response --- 20:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:56 DEBUG ----- output ----- 20:22:56 DEBUG { 20:22:56 DEBUG "status": "success", 20:22:56 DEBUG "data": { 20:22:56 DEBUG "resultType": "vector", 20:22:56 DEBUG "result": [ 20:22:56 DEBUG { 20:22:56 DEBUG "metric": { 20:22:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:56 DEBUG }, 20:22:56 DEBUG "value": [ 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG [...] 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:56 DEBUG }, 20:22:56 DEBUG "value": [ 20:22:56 DEBUG 1713640976, 20:22:56 DEBUG "2820.9333333333334" 20:22:56 DEBUG ] 20:22:56 DEBUG } 20:22:56 DEBUG ] 20:22:56 DEBUG } 20:22:56 DEBUG } 20:22:56 INFO [http_cmd]: http status code OK 20:22:56 DEBUG --- status code --- 20:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:56 DEBUG --- http response --- 20:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:56 DEBUG ----- output ----- 20:22:56 DEBUG { 20:22:56 DEBUG "status": "success", 20:22:56 DEBUG "data": { 20:22:56 DEBUG "resultType": "vector", 20:22:56 DEBUG "result": [ 20:22:56 DEBUG { 20:22:56 DEBUG "metric": { 20:22:56 DEBUG "instance": "10.142.0.107:9100" 20:22:56 DEBUG }, 20:22:56 DEBUG "value": [ 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG [...] 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG "instance": "10.142.0.43:9100" 20:22:56 DEBUG }, 20:22:56 DEBUG "value": [ 20:22:56 DEBUG 1713640976, 20:22:56 DEBUG "3742094459904" 20:22:56 DEBUG ] 20:22:56 DEBUG } 20:22:56 DEBUG ] 20:22:56 DEBUG } 20:22:56 DEBUG } 20:22:56 INFO [http_cmd]: http status code OK 20:22:56 DEBUG --- status code --- 20:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:56 DEBUG --- http response --- 20:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:56 DEBUG ----- output ----- 20:22:56 DEBUG { 20:22:56 DEBUG "status": "success", 20:22:56 DEBUG "data": { 20:22:56 DEBUG "resultType": "vector", 20:22:56 DEBUG "result": [ 20:22:56 DEBUG { 20:22:56 DEBUG "metric": { 20:22:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:22:56 DEBUG }, 20:22:56 DEBUG "value": [ 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG [...] 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG 20:22:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:22:56 DEBUG }, 20:22:56 DEBUG "value": [ 20:22:56 DEBUG 1713640976, 20:22:56 DEBUG "9.560584558210659" 20:22:56 DEBUG ] 20:22:56 DEBUG } 20:22:56 DEBUG ] 20:22:56 DEBUG } 20:22:56 DEBUG } 20:22:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640977" 20:22:57 INFO [http_cmd]: http status code OK 20:22:57 DEBUG --- status code --- 20:22:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:57 DEBUG --- http response --- 20:22:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:57 DEBUG ----- output ----- 20:22:57 DEBUG { 20:22:57 DEBUG "status": "success", 20:22:57 DEBUG "data": { 20:22:57 DEBUG "resultType": "vector", 20:22:57 DEBUG "result": [ 20:22:57 DEBUG { 20:22:57 DEBUG "metric": { 20:22:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:57 DEBUG }, 20:22:57 DEBUG "value": [ 20:22:57 DEBUG 20:22:57 DEBUG 20:22:57 DEBUG 20:22:57 DEBUG [...] 20:22:57 DEBUG 20:22:57 DEBUG 20:22:57 DEBUG 20:22:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:57 DEBUG }, 20:22:57 DEBUG "value": [ 20:22:57 DEBUG 1713640977, 20:22:57 DEBUG "2821.8" 20:22:57 DEBUG ] 20:22:57 DEBUG } 20:22:57 DEBUG ] 20:22:57 DEBUG } 20:22:57 DEBUG } 20:22:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640977" 20:22:57 INFO [http_cmd]: http status code OK 20:22:57 DEBUG --- status code --- 20:22:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:57 DEBUG --- http response --- 20:22:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:57 DEBUG ----- output ----- 20:22:57 DEBUG { 20:22:57 DEBUG "status": "success", 20:22:57 DEBUG "data": { 20:22:57 DEBUG "resultType": "vector", 20:22:57 DEBUG "result": [ 20:22:57 DEBUG { 20:22:57 DEBUG "metric": { 20:22:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:57 DEBUG }, 20:22:57 DEBUG "value": [ 20:22:57 DEBUG 20:22:57 DEBUG 20:22:57 DEBUG 20:22:57 DEBUG [...] 20:22:57 DEBUG 20:22:57 DEBUG 20:22:57 DEBUG 20:22:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:57 DEBUG }, 20:22:57 DEBUG "value": [ 20:22:57 DEBUG 1713640977, 20:22:57 DEBUG "2821.1666666666665" 20:22:57 DEBUG ] 20:22:57 DEBUG } 20:22:57 DEBUG ] 20:22:57 DEBUG } 20:22:57 DEBUG } 20:22:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640977" 20:22:57 INFO [http_cmd]: http status code OK 20:22:57 DEBUG --- status code --- 20:22:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:57 DEBUG --- http response --- 20:22:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:22:57 DEBUG ----- output ----- 20:22:57 DEBUG { 20:22:57 DEBUG "status": "success", 20:22:57 DEBUG "data": { 20:22:57 DEBUG "resultType": "vector", 20:22:57 DEBUG "result": [ 20:22:57 DEBUG { 20:22:57 DEBUG "metric": { 20:22:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:22:57 DEBUG }, 20:22:57 DEBUG "value": [ 20:22:57 DEBUG 20:22:57 DEBUG 20:22:57 DEBUG 20:22:57 DEBUG [...] 20:22:57 DEBUG 20:22:57 DEBUG 20:22:57 DEBUG 20:22:57 DEBUG "pod": "idm-b85f9d978-txpwv" 20:22:57 DEBUG }, 20:22:57 DEBUG "value": [ 20:22:57 DEBUG 1713640977, 20:22:57 DEBUG "0.003183073106668008" 20:22:57 DEBUG ] 20:22:57 DEBUG } 20:22:57 DEBUG ] 20:22:57 DEBUG } 20:22:57 DEBUG } 20:22:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640978" 20:22:58 INFO [http_cmd]: http status code OK 20:22:58 DEBUG --- status code --- 20:22:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:58 DEBUG --- http response --- 20:22:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:58 DEBUG ----- output ----- 20:22:58 DEBUG { 20:22:58 DEBUG "status": "success", 20:22:58 DEBUG "data": { 20:22:58 DEBUG "resultType": "vector", 20:22:58 DEBUG "result": [ 20:22:58 DEBUG { 20:22:58 DEBUG "metric": { 20:22:58 DEBUG "pod": "ds-idrepo-0" 20:22:58 DEBUG }, 20:22:58 DEBUG "value": [ 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG [...] 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG "pod": "ds-idrepo-2" 20:22:58 DEBUG }, 20:22:58 DEBUG "value": [ 20:22:58 DEBUG 1713640978, 20:22:58 DEBUG "0" 20:22:58 DEBUG ] 20:22:58 DEBUG } 20:22:58 DEBUG ] 20:22:58 DEBUG } 20:22:58 DEBUG } 20:22:58 INFO 20:22:58 INFO [loop_until]: kubectl --namespace=xlou top pods 20:22:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:22:58 INFO [loop_until]: OK (rc = 0) 20:22:58 DEBUG --- stdout --- 20:22:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18545m 8662Mi am-767f6b67b5-qcqz2 18763m 9072Mi am-767f6b67b5-vbltz 19961m 8672Mi ds-cts-0 20301m 11048Mi ds-cts-1 17847m 10844Mi ds-cts-2 20082m 11083Mi ds-idrepo-0 8079m 3893Mi ds-idrepo-1 7798m 3882Mi ds-idrepo-2 7945m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9503m 844Mi 20:22:58 DEBUG --- stderr --- 20:22:58 DEBUG 20:22:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640978" 20:22:58 INFO [http_cmd]: http status code OK 20:22:58 DEBUG --- status code --- 20:22:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:58 DEBUG --- http response --- 20:22:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:22:58 DEBUG ----- output ----- 20:22:58 DEBUG { 20:22:58 DEBUG "status": "success", 20:22:58 DEBUG "data": { 20:22:58 DEBUG "resultType": "vector", 20:22:58 DEBUG "result": [ 20:22:58 DEBUG { 20:22:58 DEBUG "metric": { 20:22:58 DEBUG "pod": "ds-cts-0" 20:22:58 DEBUG }, 20:22:58 DEBUG "value": [ 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG [...] 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG "pod": "ds-cts-2" 20:22:58 DEBUG }, 20:22:58 DEBUG "value": [ 20:22:58 DEBUG 1713640978, 20:22:58 DEBUG "0" 20:22:58 DEBUG ] 20:22:58 DEBUG } 20:22:58 DEBUG ] 20:22:58 DEBUG } 20:22:58 DEBUG } 20:22:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713640978" 20:22:58 INFO [http_cmd]: http status code OK 20:22:58 DEBUG --- status code --- 20:22:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:58 DEBUG --- http response --- 20:22:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:22:58 DEBUG ----- output ----- 20:22:58 DEBUG { 20:22:58 DEBUG "status": "success", 20:22:58 DEBUG "data": { 20:22:58 DEBUG "resultType": "vector", 20:22:58 DEBUG "result": [ 20:22:58 DEBUG { 20:22:58 DEBUG "metric": { 20:22:58 DEBUG "pod": "ds-idrepo-0" 20:22:58 DEBUG }, 20:22:58 DEBUG "value": [ 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG [...] 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG "pod": "ds-cts-2" 20:22:58 DEBUG }, 20:22:58 DEBUG "value": [ 20:22:58 DEBUG 1713640978, 20:22:58 DEBUG "0" 20:22:58 DEBUG ] 20:22:58 DEBUG } 20:22:58 DEBUG ] 20:22:58 DEBUG } 20:22:58 DEBUG } 20:22:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640978" 20:22:58 INFO [http_cmd]: http status code OK 20:22:58 DEBUG --- status code --- 20:22:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:58 DEBUG --- http response --- 20:22:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:22:58 DEBUG ----- output ----- 20:22:58 DEBUG { 20:22:58 DEBUG "status": "success", 20:22:58 DEBUG "data": { 20:22:58 DEBUG "resultType": "vector", 20:22:58 DEBUG "result": [ 20:22:58 DEBUG { 20:22:58 DEBUG "metric": { 20:22:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:22:58 DEBUG }, 20:22:58 DEBUG "value": [ 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG [...] 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG 20:22:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:22:58 DEBUG }, 20:22:58 DEBUG "value": [ 20:22:58 DEBUG 1713640978, 20:22:58 DEBUG "510189.5" 20:22:58 DEBUG ] 20:22:58 DEBUG } 20:22:58 DEBUG ] 20:22:58 DEBUG } 20:22:58 DEBUG } 20:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640979" 20:22:59 INFO [http_cmd]: http status code OK 20:22:59 DEBUG --- status code --- 20:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:59 DEBUG --- http response --- 20:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:59 DEBUG ----- output ----- 20:22:59 DEBUG { 20:22:59 DEBUG "status": "success", 20:22:59 DEBUG "data": { 20:22:59 DEBUG "resultType": "vector", 20:22:59 DEBUG "result": [ 20:22:59 DEBUG { 20:22:59 DEBUG "metric": { 20:22:59 DEBUG "instance": "10.142.0.107:9100" 20:22:59 DEBUG }, 20:22:59 DEBUG "value": [ 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG [...] 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG "instance": "10.142.0.43:9100" 20:22:59 DEBUG }, 20:22:59 DEBUG "value": [ 20:22:59 DEBUG 1713640979, 20:22:59 DEBUG "0.2056999999999609" 20:22:59 DEBUG ] 20:22:59 DEBUG } 20:22:59 DEBUG ] 20:22:59 DEBUG } 20:22:59 DEBUG } 20:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640979" 20:22:59 INFO [http_cmd]: http status code OK 20:22:59 DEBUG --- status code --- 20:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:59 DEBUG --- http response --- 20:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:59 DEBUG ----- output ----- 20:22:59 DEBUG { 20:22:59 DEBUG "status": "success", 20:22:59 DEBUG "data": { 20:22:59 DEBUG "resultType": "vector", 20:22:59 DEBUG "result": [ 20:22:59 DEBUG { 20:22:59 DEBUG "metric": { 20:22:59 DEBUG "instance": "10.142.0.107:9100" 20:22:59 DEBUG }, 20:22:59 DEBUG "value": [ 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG [...] 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG "instance": "10.142.0.43:9100" 20:22:59 DEBUG }, 20:22:59 DEBUG "value": [ 20:22:59 DEBUG 1713640979, 20:22:59 DEBUG "85641272.9" 20:22:59 DEBUG ] 20:22:59 DEBUG } 20:22:59 DEBUG ] 20:22:59 DEBUG } 20:22:59 DEBUG } 20:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713640979" 20:22:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640979" 20:22:59 INFO [http_cmd]: http status code OK 20:22:59 DEBUG --- status code --- 20:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:59 DEBUG --- http response --- 20:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:59 DEBUG ----- output ----- 20:22:59 DEBUG { 20:22:59 DEBUG "status": "success", 20:22:59 DEBUG "data": { 20:22:59 DEBUG "resultType": "vector", 20:22:59 DEBUG "result": [ 20:22:59 DEBUG { 20:22:59 DEBUG "metric": { 20:22:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:22:59 DEBUG }, 20:22:59 DEBUG "value": [ 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG [...] 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:22:59 DEBUG }, 20:22:59 DEBUG "value": [ 20:22:59 DEBUG 1713640979, 20:22:59 DEBUG "983916544" 20:22:59 DEBUG ] 20:22:59 DEBUG } 20:22:59 DEBUG ] 20:22:59 DEBUG } 20:22:59 DEBUG } 20:22:59 INFO [http_cmd]: http status code OK 20:22:59 DEBUG --- status code --- 20:22:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:22:59 DEBUG --- http response --- 20:22:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:22:59 DEBUG ----- output ----- 20:22:59 DEBUG { 20:22:59 DEBUG "status": "success", 20:22:59 DEBUG "data": { 20:22:59 DEBUG "resultType": "vector", 20:22:59 DEBUG "result": [ 20:22:59 DEBUG { 20:22:59 DEBUG "metric": { 20:22:59 DEBUG "instance": "10.142.0.107:9100" 20:22:59 DEBUG }, 20:22:59 DEBUG "value": [ 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG [...] 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG 20:22:59 DEBUG "instance": "10.142.0.43:9100" 20:22:59 DEBUG }, 20:22:59 DEBUG "value": [ 20:22:59 DEBUG 1713640979, 20:22:59 DEBUG "85645124.43333334" 20:22:59 DEBUG ] 20:22:59 DEBUG } 20:22:59 DEBUG ] 20:22:59 DEBUG } 20:22:59 DEBUG } 20:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640980" 20:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640980" 20:23:00 INFO [http_cmd]: http status code OK 20:23:00 DEBUG --- status code --- 20:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:00 DEBUG --- http response --- 20:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:23:00 DEBUG ----- output ----- 20:23:00 DEBUG { 20:23:00 DEBUG "status": "success", 20:23:00 DEBUG "data": { 20:23:00 DEBUG "resultType": "vector", 20:23:00 DEBUG "result": [ 20:23:00 DEBUG { 20:23:00 DEBUG "metric": { 20:23:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:23:00 DEBUG }, 20:23:00 DEBUG "value": [ 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG [...] 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:23:00 DEBUG }, 20:23:00 DEBUG "value": [ 20:23:00 DEBUG 1713640980, 20:23:00 DEBUG "0" 20:23:00 DEBUG ] 20:23:00 DEBUG } 20:23:00 DEBUG ] 20:23:00 DEBUG } 20:23:00 DEBUG } 20:23:00 INFO [http_cmd]: http status code OK 20:23:00 DEBUG --- status code --- 20:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:00 DEBUG --- http response --- 20:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:23:00 DEBUG ----- output ----- 20:23:00 DEBUG { 20:23:00 DEBUG "status": "success", 20:23:00 DEBUG "data": { 20:23:00 DEBUG "resultType": "vector", 20:23:00 DEBUG "result": [ 20:23:00 DEBUG { 20:23:00 DEBUG "metric": { 20:23:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:23:00 DEBUG }, 20:23:00 DEBUG "value": [ 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG [...] 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:23:00 DEBUG }, 20:23:00 DEBUG "value": [ 20:23:00 DEBUG 1713640980, 20:23:00 DEBUG "0" 20:23:00 DEBUG ] 20:23:00 DEBUG } 20:23:00 DEBUG ] 20:23:00 DEBUG } 20:23:00 DEBUG } 20:23:00 INFO 20:23:00 INFO [loop_until]: kubectl --namespace=xlou top node 20:23:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640980" 20:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713640980" 20:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640980" 20:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713640980" 20:23:00 INFO [http_cmd]: http status code OK 20:23:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713640980" 20:23:00 DEBUG --- status code --- 20:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:00 DEBUG --- http response --- 20:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:23:00 DEBUG ----- output ----- 20:23:00 INFO [http_cmd]: http status code OK 20:23:00 DEBUG { 20:23:00 DEBUG --- status code --- 20:23:00 DEBUG "status": "success", 20:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:00 DEBUG "data": { 20:23:00 DEBUG --- http response --- 20:23:00 INFO [http_cmd]: http status code OK 20:23:00 DEBUG "resultType": "vector", 20:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:23:00 DEBUG --- status code --- 20:23:00 DEBUG "result": [ 20:23:00 DEBUG ----- output ----- 20:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:00 INFO [http_cmd]: http status code OK 20:23:00 DEBUG { 20:23:00 DEBUG "metric": { 20:23:00 DEBUG --- http response --- 20:23:00 DEBUG --- status code --- 20:23:00 DEBUG "pod": "ds-cts-0" 20:23:00 DEBUG { 20:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:00 DEBUG }, 20:23:00 DEBUG "status": "success", 20:23:00 DEBUG "data": { 20:23:00 DEBUG "resultType": "vector", 20:23:00 DEBUG "value": [ 20:23:00 DEBUG "result": [ 20:23:00 DEBUG --- http response --- 20:23:00 DEBUG ----- output ----- 20:23:00 DEBUG 20:23:00 DEBUG { 20:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:23:00 DEBUG { 20:23:00 DEBUG 20:23:00 DEBUG "metric": { 20:23:00 DEBUG ----- output ----- 20:23:00 DEBUG "status": "success", 20:23:00 DEBUG 20:23:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:23:00 DEBUG { 20:23:00 DEBUG "data": { 20:23:00 DEBUG [...] 20:23:00 DEBUG }, 20:23:00 DEBUG "status": "success", 20:23:00 DEBUG "resultType": "vector", 20:23:00 DEBUG "result": [ 20:23:00 DEBUG "value": [ 20:23:00 DEBUG 20:23:00 DEBUG "data": { 20:23:00 DEBUG { 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG "resultType": "vector", 20:23:00 DEBUG "result": [ 20:23:00 DEBUG 20:23:00 DEBUG [...] 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG "pod": "ds-cts-2" 20:23:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:23:00 DEBUG }, 20:23:00 DEBUG "value": [ 20:23:00 DEBUG "metric": { 20:23:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:23:00 DEBUG }, 20:23:00 DEBUG "value": [ 20:23:00 DEBUG 20:23:00 DEBUG { 20:23:00 DEBUG 20:23:00 INFO [http_cmd]: http status code OK 20:23:00 DEBUG --- status code --- 20:23:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:00 DEBUG --- http response --- 20:23:00 DEBUG 20:23:00 DEBUG [...] 20:23:00 DEBUG "metric": { 20:23:00 DEBUG 1713640980, 20:23:00 DEBUG "0.159" 20:23:00 DEBUG ] 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG } 20:23:00 DEBUG ] 20:23:00 DEBUG } 20:23:00 DEBUG 20:23:00 DEBUG "pod": "idm-b85f9d978-txpwv" 20:23:00 DEBUG } 20:23:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:23:00 DEBUG ----- output ----- 20:23:00 DEBUG { 20:23:00 DEBUG "pod": "ds-cts-0" 20:23:00 DEBUG }, 20:23:00 DEBUG "status": "success", 20:23:00 DEBUG "data": { 20:23:00 DEBUG "resultType": "vector", 20:23:00 DEBUG "result": [ 20:23:00 DEBUG { 20:23:00 DEBUG "value": [ 20:23:00 DEBUG "metric": { 20:23:00 DEBUG "instance": "10.142.0.107:9100" 20:23:00 DEBUG }, 20:23:00 DEBUG 20:23:00 DEBUG "value": [ 20:23:00 DEBUG }, 20:23:00 DEBUG }, 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG "value": [ 20:23:00 DEBUG "value": [ 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG 1713640980, 20:23:00 DEBUG 1713640980, 20:23:00 DEBUG [...] 20:23:00 DEBUG 20:23:00 DEBUG [...] 20:23:00 DEBUG "0" 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG "33.5" 20:23:00 DEBUG ] 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG ] 20:23:00 DEBUG } 20:23:00 DEBUG 20:23:00 DEBUG 20:23:00 DEBUG } 20:23:00 DEBUG ] 20:23:00 DEBUG "pod": "ds-cts-2" 20:23:00 DEBUG "instance": "10.142.0.43:9100" 20:23:00 DEBUG ] 20:23:00 DEBUG } 20:23:00 DEBUG }, 20:23:00 DEBUG }, 20:23:00 DEBUG } 20:23:00 DEBUG } 20:23:00 DEBUG "value": [ 20:23:00 DEBUG "value": [ 20:23:00 DEBUG } 20:23:00 DEBUG 1713640980, 20:23:00 DEBUG 1713640980, 20:23:00 DEBUG "0.008566666666666663" 20:23:00 DEBUG "0.07099999999999985" 20:23:00 DEBUG ] 20:23:00 DEBUG } 20:23:00 DEBUG ] 20:23:00 DEBUG ] 20:23:00 DEBUG } 20:23:00 DEBUG } 20:23:00 DEBUG ] 20:23:00 DEBUG } 20:23:00 DEBUG } 20:23:00 DEBUG } 20:23:00 INFO [loop_until]: OK (rc = 0) 20:23:00 DEBUG --- stdout --- 20:23:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6214Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8174m 27% 7640Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18140m 60% 11380Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3345Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19669m 65% 14234Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 149m 0% 5194Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17880m 59% 11961Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15594m 52% 2969Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20826m 69% 10884Mi 9% gke-xlou-cdm-ds-0f601751-j488 8710m 29% 7219Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18590m 62% 15884Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8645m 28% 7854Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19939m 66% 13757Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9741m 61% 6398Mi 10% 20:23:00 DEBUG --- stderr --- 20:23:00 DEBUG 20:23:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641034" 20:23:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713641034" 20:23:54 ERROR [http_cmd]: ERROR 20:23:54 INFO [http_cmd]: http status code OK 20:23:54 DEBUG --- status code --- 20:23:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:54 ERROR --- status code --- 20:23:54 DEBUG --- http response --- 20:23:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:23:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:23:54 ERROR --- http response --- 20:23:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:23:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:23:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641035" 20:23:55 INFO [http_cmd]: http status code OK 20:23:55 DEBUG --- status code --- 20:23:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:55 DEBUG --- http response --- 20:23:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:23:55 DEBUG ----- output ----- 20:23:55 DEBUG { 20:23:55 DEBUG "status": "success", 20:23:55 DEBUG "data": { 20:23:55 DEBUG "resultType": "vector", 20:23:55 DEBUG "result": [ 20:23:55 DEBUG { 20:23:55 DEBUG "metric": { 20:23:55 DEBUG "pod": "ds-idrepo-0" 20:23:55 DEBUG }, 20:23:55 DEBUG "value": [ 20:23:55 DEBUG 20:23:55 DEBUG 20:23:55 DEBUG 20:23:55 DEBUG [...] 20:23:55 DEBUG 20:23:55 DEBUG 20:23:55 DEBUG 20:23:55 DEBUG "pod": "ds-idrepo-2" 20:23:55 DEBUG }, 20:23:55 DEBUG "value": [ 20:23:55 DEBUG 1713641035, 20:23:55 DEBUG "0" 20:23:55 DEBUG ] 20:23:55 DEBUG } 20:23:55 DEBUG ] 20:23:55 DEBUG } 20:23:55 DEBUG } 20:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641036" 20:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641036" 20:23:56 INFO [http_cmd]: http status code OK 20:23:56 DEBUG --- status code --- 20:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:56 DEBUG --- http response --- 20:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:23:56 DEBUG ----- output ----- 20:23:56 DEBUG { 20:23:56 DEBUG "status": "success", 20:23:56 DEBUG "data": { 20:23:56 DEBUG "resultType": "vector", 20:23:56 DEBUG "result": [ 20:23:56 DEBUG { 20:23:56 DEBUG "metric": { 20:23:56 DEBUG "pod": "ds-idrepo-0" 20:23:56 DEBUG }, 20:23:56 INFO [http_cmd]: http status code OK 20:23:56 DEBUG "value": [ 20:23:56 DEBUG --- status code --- 20:23:56 DEBUG 20:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:56 DEBUG 20:23:56 DEBUG --- http response --- 20:23:56 DEBUG 20:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:23:56 DEBUG [...] 20:23:56 DEBUG ----- output ----- 20:23:56 DEBUG 20:23:56 DEBUG { 20:23:56 DEBUG 20:23:56 DEBUG "status": "success", 20:23:56 DEBUG 20:23:56 DEBUG "data": { 20:23:56 DEBUG "pod": "ds-idrepo-2" 20:23:56 DEBUG "resultType": "vector", 20:23:56 DEBUG }, 20:23:56 DEBUG "result": [ 20:23:56 DEBUG "value": [ 20:23:56 DEBUG { 20:23:56 DEBUG 1713641036, 20:23:56 DEBUG "metric": { 20:23:56 DEBUG "0" 20:23:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:23:56 DEBUG ] 20:23:56 DEBUG }, 20:23:56 DEBUG } 20:23:56 DEBUG "value": [ 20:23:56 DEBUG ] 20:23:56 DEBUG 20:23:56 DEBUG } 20:23:56 DEBUG 20:23:56 DEBUG } 20:23:56 DEBUG 20:23:56 DEBUG [...] 20:23:56 DEBUG 20:23:56 DEBUG 20:23:56 DEBUG 20:23:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:23:56 DEBUG }, 20:23:56 DEBUG "value": [ 20:23:56 DEBUG 1713641036, 20:23:56 DEBUG "2843.6666666666665" 20:23:56 DEBUG ] 20:23:56 DEBUG } 20:23:56 DEBUG ] 20:23:56 DEBUG } 20:23:56 DEBUG } 20:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641036" 20:23:56 INFO [http_cmd]: http status code OK 20:23:56 DEBUG --- status code --- 20:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:56 DEBUG --- http response --- 20:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:23:56 DEBUG ----- output ----- 20:23:56 DEBUG { 20:23:56 DEBUG "status": "success", 20:23:56 DEBUG "data": { 20:23:56 DEBUG "resultType": "vector", 20:23:56 DEBUG "result": [ 20:23:56 DEBUG { 20:23:56 DEBUG "metric": { 20:23:56 DEBUG "pod": "ds-idrepo-1" 20:23:56 DEBUG }, 20:23:56 DEBUG "value": [ 20:23:56 DEBUG 20:23:56 DEBUG 20:23:56 DEBUG 20:23:56 DEBUG [...] 20:23:56 DEBUG 20:23:56 DEBUG 20:23:56 DEBUG 20:23:56 DEBUG "pod": "ds-cts-0" 20:23:56 DEBUG }, 20:23:56 DEBUG "value": [ 20:23:56 DEBUG 1713641036, 20:23:56 DEBUG "45312106.16168312" 20:23:56 DEBUG ] 20:23:56 DEBUG } 20:23:56 DEBUG ] 20:23:56 DEBUG } 20:23:56 DEBUG } 20:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641036" 20:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713641036" 20:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641036" 20:23:56 INFO [http_cmd]: http status code OK 20:23:56 DEBUG --- status code --- 20:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:56 DEBUG --- http response --- 20:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:23:56 DEBUG ----- output ----- 20:23:56 DEBUG { 20:23:56 DEBUG "status": "success", 20:23:56 DEBUG "data": { 20:23:56 DEBUG "resultType": "vector", 20:23:56 DEBUG "result": [ 20:23:56 DEBUG { 20:23:56 DEBUG "metric": { 20:23:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:23:56 DEBUG }, 20:23:56 DEBUG "value": [ 20:23:56 DEBUG 20:23:56 DEBUG 20:23:56 DEBUG 20:23:56 DEBUG [...] 20:23:56 DEBUG 20:23:56 DEBUG 20:23:56 DEBUG 20:23:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:23:56 DEBUG }, 20:23:56 DEBUG "value": [ 20:23:56 DEBUG 1713641036, 20:23:56 DEBUG "2834.866666666667" 20:23:56 DEBUG ] 20:23:56 DEBUG } 20:23:56 DEBUG ] 20:23:56 DEBUG } 20:23:56 DEBUG } 20:23:56 INFO [http_cmd]: http status code OK 20:23:56 DEBUG --- status code --- 20:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:56 DEBUG --- http response --- 20:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:23:56 DEBUG ----- output ----- 20:23:56 DEBUG { 20:23:56 DEBUG "status": "success", 20:23:56 DEBUG "data": { 20:23:56 DEBUG "resultType": "vector", 20:23:56 INFO [http_cmd]: http status code OK 20:23:56 DEBUG "result": [ 20:23:56 DEBUG --- status code --- 20:23:56 DEBUG { 20:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:56 DEBUG "metric": { 20:23:56 DEBUG --- http response --- 20:23:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:23:56 DEBUG }, 20:23:56 DEBUG "value": [ 20:23:56 DEBUG 20:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:23:56 DEBUG 20:23:56 DEBUG ----- output ----- 20:23:56 DEBUG 20:23:56 DEBUG { 20:23:56 DEBUG [...] 20:23:56 DEBUG "status": "success", 20:23:56 DEBUG 20:23:56 DEBUG "data": { 20:23:56 DEBUG 20:23:56 DEBUG "resultType": "vector", 20:23:56 DEBUG 20:23:56 DEBUG "result": [ 20:23:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:23:56 DEBUG { 20:23:56 DEBUG }, 20:23:56 DEBUG "metric": { 20:23:56 DEBUG "value": [ 20:23:56 DEBUG "instance": "10.142.0.107:9100" 20:23:56 DEBUG 1713641036, 20:23:56 DEBUG }, 20:23:56 DEBUG "9.568520054181024" 20:23:56 DEBUG "value": [ 20:23:56 DEBUG ] 20:23:56 DEBUG 20:23:56 DEBUG } 20:23:56 DEBUG 20:23:56 DEBUG ] 20:23:56 DEBUG 20:23:56 DEBUG } 20:23:56 DEBUG [...] 20:23:56 DEBUG } 20:23:56 DEBUG 20:23:56 DEBUG 20:23:56 DEBUG 20:23:56 DEBUG "instance": "10.142.0.43:9100" 20:23:56 DEBUG }, 20:23:56 DEBUG "value": [ 20:23:56 DEBUG 1713641036, 20:23:56 DEBUG "3746912707584" 20:23:56 DEBUG ] 20:23:56 DEBUG } 20:23:56 DEBUG ] 20:23:56 DEBUG } 20:23:56 DEBUG } 20:23:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641037" 20:23:57 INFO [http_cmd]: http status code OK 20:23:57 DEBUG --- status code --- 20:23:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:57 DEBUG --- http response --- 20:23:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:23:57 DEBUG ----- output ----- 20:23:57 DEBUG { 20:23:57 DEBUG "status": "success", 20:23:57 DEBUG "data": { 20:23:57 DEBUG "resultType": "vector", 20:23:57 DEBUG "result": [ 20:23:57 DEBUG { 20:23:57 DEBUG "metric": { 20:23:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:23:57 DEBUG }, 20:23:57 DEBUG "value": [ 20:23:57 DEBUG 20:23:57 DEBUG 20:23:57 DEBUG 20:23:57 DEBUG [...] 20:23:57 DEBUG 20:23:57 DEBUG 20:23:57 DEBUG 20:23:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:23:57 DEBUG }, 20:23:57 DEBUG "value": [ 20:23:57 DEBUG 1713641037, 20:23:57 DEBUG "2834.5666666666666" 20:23:57 DEBUG ] 20:23:57 DEBUG } 20:23:57 DEBUG ] 20:23:57 DEBUG } 20:23:57 DEBUG } 20:23:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641037" 20:23:57 INFO [http_cmd]: http status code OK 20:23:57 DEBUG --- status code --- 20:23:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:57 DEBUG --- http response --- 20:23:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:23:57 DEBUG ----- output ----- 20:23:57 DEBUG { 20:23:57 DEBUG "status": "success", 20:23:57 DEBUG "data": { 20:23:57 DEBUG "resultType": "vector", 20:23:57 DEBUG "result": [ 20:23:57 DEBUG { 20:23:57 DEBUG "metric": { 20:23:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:23:57 DEBUG }, 20:23:57 DEBUG "value": [ 20:23:57 DEBUG 20:23:57 DEBUG 20:23:57 DEBUG 20:23:57 DEBUG [...] 20:23:57 DEBUG 20:23:57 DEBUG 20:23:57 DEBUG 20:23:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:23:57 DEBUG }, 20:23:57 DEBUG "value": [ 20:23:57 DEBUG 1713641037, 20:23:57 DEBUG "2835.4" 20:23:57 DEBUG ] 20:23:57 DEBUG } 20:23:57 DEBUG ] 20:23:57 DEBUG } 20:23:57 DEBUG } 20:23:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641037" 20:23:57 INFO [http_cmd]: http status code OK 20:23:57 DEBUG --- status code --- 20:23:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:57 DEBUG --- http response --- 20:23:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:23:57 DEBUG ----- output ----- 20:23:57 DEBUG { 20:23:57 DEBUG "status": "success", 20:23:57 DEBUG "data": { 20:23:57 DEBUG "resultType": "vector", 20:23:57 DEBUG "result": [ 20:23:57 DEBUG { 20:23:57 DEBUG "metric": { 20:23:57 DEBUG "pod": "ds-idrepo-1" 20:23:57 DEBUG }, 20:23:57 DEBUG "value": [ 20:23:57 DEBUG 20:23:57 DEBUG 20:23:57 DEBUG 20:23:57 DEBUG [...] 20:23:57 DEBUG 20:23:57 DEBUG 20:23:57 DEBUG 20:23:57 DEBUG "pod": "ds-cts-0" 20:23:57 DEBUG }, 20:23:57 DEBUG "value": [ 20:23:57 DEBUG 1713641037, 20:23:57 DEBUG "12.138771136412828" 20:23:57 DEBUG ] 20:23:57 DEBUG } 20:23:57 DEBUG ] 20:23:57 DEBUG } 20:23:57 DEBUG } 20:23:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641038" 20:23:58 INFO [http_cmd]: http status code OK 20:23:58 DEBUG --- status code --- 20:23:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:58 DEBUG --- http response --- 20:23:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:23:58 DEBUG ----- output ----- 20:23:58 DEBUG { 20:23:58 DEBUG "status": "success", 20:23:58 DEBUG "data": { 20:23:58 DEBUG "resultType": "vector", 20:23:58 DEBUG "result": [ 20:23:58 DEBUG { 20:23:58 DEBUG "metric": { 20:23:58 DEBUG "pod": "ds-idrepo-0" 20:23:58 DEBUG }, 20:23:58 DEBUG "value": [ 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG [...] 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG "pod": "ds-idrepo-2" 20:23:58 DEBUG }, 20:23:58 DEBUG "value": [ 20:23:58 DEBUG 1713641038, 20:23:58 DEBUG "0" 20:23:58 DEBUG ] 20:23:58 DEBUG } 20:23:58 DEBUG ] 20:23:58 DEBUG } 20:23:58 DEBUG } 20:23:58 INFO 20:23:58 INFO [loop_until]: kubectl --namespace=xlou top pods 20:23:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:23:58 INFO [loop_until]: OK (rc = 0) 20:23:58 DEBUG --- stdout --- 20:23:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18950m 8662Mi am-767f6b67b5-qcqz2 18501m 9071Mi am-767f6b67b5-vbltz 20078m 8673Mi ds-cts-0 19290m 11035Mi ds-cts-1 19089m 10847Mi ds-cts-2 20457m 11098Mi ds-idrepo-0 7862m 3893Mi ds-idrepo-1 7784m 3883Mi ds-idrepo-2 7887m 3828Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9293m 847Mi 20:23:58 DEBUG --- stderr --- 20:23:58 DEBUG 20:23:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641038" 20:23:58 INFO [http_cmd]: http status code OK 20:23:58 DEBUG --- status code --- 20:23:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:58 DEBUG --- http response --- 20:23:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:23:58 DEBUG ----- output ----- 20:23:58 DEBUG { 20:23:58 DEBUG "status": "success", 20:23:58 DEBUG "data": { 20:23:58 DEBUG "resultType": "vector", 20:23:58 DEBUG "result": [ 20:23:58 DEBUG { 20:23:58 DEBUG "metric": { 20:23:58 DEBUG "pod": "ds-cts-0" 20:23:58 DEBUG }, 20:23:58 DEBUG "value": [ 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG [...] 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG "pod": "ds-cts-2" 20:23:58 DEBUG }, 20:23:58 DEBUG "value": [ 20:23:58 DEBUG 1713641038, 20:23:58 DEBUG "0" 20:23:58 DEBUG ] 20:23:58 DEBUG } 20:23:58 DEBUG ] 20:23:58 DEBUG } 20:23:58 DEBUG } 20:23:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713641038" 20:23:58 INFO [http_cmd]: http status code OK 20:23:58 DEBUG --- status code --- 20:23:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:58 DEBUG --- http response --- 20:23:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:23:58 DEBUG ----- output ----- 20:23:58 DEBUG { 20:23:58 DEBUG "status": "success", 20:23:58 DEBUG "data": { 20:23:58 DEBUG "resultType": "vector", 20:23:58 DEBUG "result": [ 20:23:58 DEBUG { 20:23:58 DEBUG "metric": { 20:23:58 DEBUG "pod": "ds-idrepo-0" 20:23:58 DEBUG }, 20:23:58 DEBUG "value": [ 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG [...] 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG "pod": "ds-cts-2" 20:23:58 DEBUG }, 20:23:58 DEBUG "value": [ 20:23:58 DEBUG 1713641038, 20:23:58 DEBUG "0" 20:23:58 DEBUG ] 20:23:58 DEBUG } 20:23:58 DEBUG ] 20:23:58 DEBUG } 20:23:58 DEBUG } 20:23:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641038" 20:23:58 INFO [http_cmd]: http status code OK 20:23:58 DEBUG --- status code --- 20:23:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:58 DEBUG --- http response --- 20:23:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:23:58 DEBUG ----- output ----- 20:23:58 DEBUG { 20:23:58 DEBUG "status": "success", 20:23:58 DEBUG "data": { 20:23:58 DEBUG "resultType": "vector", 20:23:58 DEBUG "result": [ 20:23:58 DEBUG { 20:23:58 DEBUG "metric": { 20:23:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:23:58 DEBUG }, 20:23:58 DEBUG "value": [ 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG [...] 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG 20:23:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:23:58 DEBUG }, 20:23:58 DEBUG "value": [ 20:23:58 DEBUG 1713641038, 20:23:58 DEBUG "509944" 20:23:58 DEBUG ] 20:23:58 DEBUG } 20:23:58 DEBUG ] 20:23:58 DEBUG } 20:23:58 DEBUG } 20:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641039" 20:23:59 INFO [http_cmd]: http status code OK 20:23:59 DEBUG --- status code --- 20:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:59 DEBUG --- http response --- 20:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:23:59 DEBUG ----- output ----- 20:23:59 DEBUG { 20:23:59 DEBUG "status": "success", 20:23:59 DEBUG "data": { 20:23:59 DEBUG "resultType": "vector", 20:23:59 DEBUG "result": [ 20:23:59 DEBUG { 20:23:59 DEBUG "metric": { 20:23:59 DEBUG "instance": "10.142.0.107:9100" 20:23:59 DEBUG }, 20:23:59 DEBUG "value": [ 20:23:59 DEBUG 20:23:59 DEBUG 20:23:59 DEBUG 20:23:59 DEBUG [...] 20:23:59 DEBUG 20:23:59 DEBUG 20:23:59 DEBUG 20:23:59 DEBUG "instance": "10.142.0.43:9100" 20:23:59 DEBUG }, 20:23:59 DEBUG "value": [ 20:23:59 DEBUG 1713641039, 20:23:59 DEBUG "0.2820000000000107" 20:23:59 DEBUG ] 20:23:59 DEBUG } 20:23:59 DEBUG ] 20:23:59 DEBUG } 20:23:59 DEBUG } 20:23:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641039" 20:23:59 INFO [http_cmd]: http status code OK 20:23:59 DEBUG --- status code --- 20:23:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:23:59 DEBUG --- http response --- 20:23:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:23:59 DEBUG ----- output ----- 20:23:59 DEBUG { 20:23:59 DEBUG "status": "success", 20:23:59 DEBUG "data": { 20:23:59 DEBUG "resultType": "vector", 20:23:59 DEBUG "result": [ 20:23:59 DEBUG { 20:23:59 DEBUG "metric": { 20:23:59 DEBUG "instance": "10.142.0.107:9100" 20:23:59 DEBUG }, 20:23:59 DEBUG "value": [ 20:23:59 DEBUG 20:23:59 DEBUG 20:23:59 DEBUG 20:23:59 DEBUG [...] 20:23:59 DEBUG 20:23:59 DEBUG 20:23:59 DEBUG 20:23:59 DEBUG "instance": "10.142.0.43:9100" 20:23:59 DEBUG }, 20:23:59 DEBUG "value": [ 20:23:59 DEBUG 1713641039, 20:23:59 DEBUG "85771218.9" 20:23:59 DEBUG ] 20:23:59 DEBUG } 20:23:59 DEBUG ] 20:23:59 DEBUG } 20:23:59 DEBUG } 20:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713641040" 20:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641040" 20:24:00 INFO [http_cmd]: http status code OK 20:24:00 DEBUG --- status code --- 20:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:00 DEBUG --- http response --- 20:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:24:00 DEBUG ----- output ----- 20:24:00 DEBUG { 20:24:00 DEBUG "status": "success", 20:24:00 DEBUG "data": { 20:24:00 DEBUG "resultType": "vector", 20:24:00 DEBUG "result": [ 20:24:00 DEBUG { 20:24:00 DEBUG "metric": { 20:24:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:24:00 DEBUG }, 20:24:00 DEBUG "value": [ 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG [...] 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:24:00 DEBUG }, 20:24:00 DEBUG "value": [ 20:24:00 DEBUG 1713641040, 20:24:00 DEBUG "986501120" 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG } 20:24:00 INFO [http_cmd]: http status code OK 20:24:00 DEBUG --- status code --- 20:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:00 DEBUG --- http response --- 20:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:24:00 DEBUG ----- output ----- 20:24:00 DEBUG { 20:24:00 DEBUG "status": "success", 20:24:00 DEBUG "data": { 20:24:00 DEBUG "resultType": "vector", 20:24:00 DEBUG "result": [ 20:24:00 DEBUG { 20:24:00 DEBUG "metric": { 20:24:00 DEBUG "instance": "10.142.0.107:9100" 20:24:00 DEBUG }, 20:24:00 DEBUG "value": [ 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG [...] 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG "instance": "10.142.0.43:9100" 20:24:00 DEBUG }, 20:24:00 DEBUG "value": [ 20:24:00 DEBUG 1713641040, 20:24:00 DEBUG "85774663.73333333" 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG } 20:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641040" 20:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641040" 20:24:00 INFO [http_cmd]: http status code OK 20:24:00 DEBUG --- status code --- 20:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:00 DEBUG --- http response --- 20:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:24:00 DEBUG ----- output ----- 20:24:00 DEBUG { 20:24:00 DEBUG "status": "success", 20:24:00 DEBUG "data": { 20:24:00 DEBUG "resultType": "vector", 20:24:00 DEBUG "result": [ 20:24:00 DEBUG { 20:24:00 DEBUG "metric": { 20:24:00 DEBUG "pod": "ds-idrepo-1" 20:24:00 DEBUG }, 20:24:00 DEBUG "value": [ 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG [...] 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG "pod": "ds-cts-0" 20:24:00 DEBUG }, 20:24:00 DEBUG "value": [ 20:24:00 DEBUG 1713641040, 20:24:00 DEBUG "676846.4743276284" 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG } 20:24:00 INFO [http_cmd]: http status code OK 20:24:00 DEBUG --- status code --- 20:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:00 DEBUG --- http response --- 20:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:24:00 DEBUG ----- output ----- 20:24:00 DEBUG { 20:24:00 DEBUG "status": "success", 20:24:00 DEBUG "data": { 20:24:00 DEBUG "resultType": "vector", 20:24:00 DEBUG "result": [ 20:24:00 DEBUG { 20:24:00 DEBUG "metric": { 20:24:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:24:00 DEBUG }, 20:24:00 DEBUG "value": [ 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG [...] 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:24:00 DEBUG }, 20:24:00 DEBUG "value": [ 20:24:00 DEBUG 1713641040, 20:24:00 DEBUG "0" 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG } 20:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713641040" 20:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641040" 20:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641040" 20:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641040" 20:24:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641040" 20:24:00 INFO [http_cmd]: http status code OK 20:24:00 DEBUG --- status code --- 20:24:00 INFO [http_cmd]: http status code OK 20:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:00 DEBUG --- status code --- 20:24:00 DEBUG --- http response --- 20:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:24:00 DEBUG --- http response --- 20:24:00 DEBUG ----- output ----- 20:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:24:00 DEBUG { 20:24:00 INFO [http_cmd]: http status code OK 20:24:00 DEBUG ----- output ----- 20:24:00 DEBUG "status": "success", 20:24:00 DEBUG --- status code --- 20:24:00 DEBUG { 20:24:00 INFO [http_cmd]: http status code OK 20:24:00 DEBUG "data": { 20:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:00 DEBUG "status": "success", 20:24:00 DEBUG --- status code --- 20:24:00 DEBUG "resultType": "vector", 20:24:00 DEBUG --- http response --- 20:24:00 DEBUG "data": { 20:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:00 DEBUG "result": [ 20:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:24:00 DEBUG "resultType": "vector", 20:24:00 DEBUG --- http response --- 20:24:00 DEBUG { 20:24:00 DEBUG ----- output ----- 20:24:00 DEBUG "result": [ 20:24:00 INFO [http_cmd]: http status code OK 20:24:00 DEBUG "metric": { 20:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:24:00 DEBUG { 20:24:00 DEBUG { 20:24:00 DEBUG --- status code --- 20:24:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:24:00 DEBUG ----- output ----- 20:24:00 DEBUG { 20:24:00 DEBUG "metric": { 20:24:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:00 DEBUG }, 20:24:00 DEBUG "status": "success", 20:24:00 DEBUG "status": "success", 20:24:00 DEBUG "pod": "ds-idrepo-1" 20:24:00 DEBUG --- http response --- 20:24:00 DEBUG "value": [ 20:24:00 DEBUG "data": { 20:24:00 DEBUG "data": { 20:24:00 DEBUG }, 20:24:00 DEBUG 20:24:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:24:00 DEBUG "resultType": "vector", 20:24:00 DEBUG "resultType": "vector", 20:24:00 DEBUG "value": [ 20:24:00 DEBUG 20:24:00 DEBUG ----- output ----- 20:24:00 DEBUG "result": [ 20:24:00 DEBUG "result": [ 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG { 20:24:00 DEBUG { 20:24:00 DEBUG { 20:24:00 DEBUG 20:24:00 DEBUG [...] 20:24:00 DEBUG "status": "success", 20:24:00 DEBUG "metric": { 20:24:00 DEBUG "metric": { 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG "data": { 20:24:00 DEBUG "pod": "ds-cts-0" 20:24:00 DEBUG "pod": "ds-cts-0" 20:24:00 DEBUG [...] 20:24:00 DEBUG 20:24:00 DEBUG "resultType": "vector", 20:24:00 DEBUG "result": [ 20:24:00 DEBUG }, 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG }, 20:24:00 DEBUG { 20:24:00 DEBUG "value": [ 20:24:00 DEBUG "pod": "ds-cts-0" 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG "metric": { 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:24:00 DEBUG "value": [ 20:24:00 DEBUG "instance": "10.142.0.107:9100" 20:24:00 DEBUG 20:24:00 DEBUG }, 20:24:00 DEBUG }, 20:24:00 DEBUG 20:24:00 DEBUG }, 20:24:00 DEBUG [...] 20:24:00 DEBUG "value": [ 20:24:00 DEBUG "value": [ 20:24:00 DEBUG 20:24:00 DEBUG "value": [ 20:24:00 DEBUG 20:24:00 DEBUG 1713641040, 20:24:00 DEBUG 1713641040, 20:24:00 DEBUG "0.159" 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG "15028064.766748166" 20:24:00 DEBUG ] 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG [...] 20:24:00 DEBUG [...] 20:24:00 DEBUG 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG } 20:24:00 DEBUG ] 20:24:00 DEBUG 20:24:00 DEBUG 20:24:00 DEBUG "pod": "ds-cts-2" 20:24:00 DEBUG }, 20:24:00 DEBUG } 20:24:00 DEBUG 20:24:00 DEBUG "pod": "ds-cts-2" 20:24:00 DEBUG }, 20:24:00 DEBUG "value": [ 20:24:00 DEBUG } 20:24:00 DEBUG "instance": "10.142.0.43:9100" 20:24:00 DEBUG "value": [ 20:24:00 DEBUG 1713641040, 20:24:00 DEBUG } 20:24:00 DEBUG }, 20:24:00 DEBUG 1713641040, 20:24:00 DEBUG "0.0005666666666666667" 20:24:00 DEBUG "value": [ 20:24:00 INFO 20:24:00 DEBUG "31.099999999999998" 20:24:00 DEBUG ] 20:24:00 DEBUG 1713641040, 20:24:00 INFO [loop_until]: kubectl --namespace=xlou top node 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG "0.09599999999999984" 20:24:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:24:00 DEBUG ] 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG } 20:24:00 DEBUG } 20:24:00 DEBUG ] 20:24:00 DEBUG ] 20:24:00 DEBUG } 20:24:00 DEBUG } 20:24:00 DEBUG } 20:24:00 DEBUG } 20:24:00 DEBUG } 20:24:00 INFO [loop_until]: OK (rc = 0) 20:24:00 DEBUG --- stdout --- 20:24:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6215Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8186m 27% 7637Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18393m 61% 11374Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3343Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19707m 66% 14233Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5197Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18614m 62% 11961Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15780m 52% 2982Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20912m 70% 10891Mi 9% gke-xlou-cdm-ds-0f601751-j488 8740m 29% 7228Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19079m 63% 15885Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8664m 29% 7869Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20236m 67% 13761Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9807m 61% 6396Mi 10% 20:24:00 DEBUG --- stderr --- 20:24:00 DEBUG 20:24:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713641094" 20:24:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641094" 20:24:54 ERROR [http_cmd]: ERROR 20:24:54 INFO [http_cmd]: http status code OK 20:24:54 ERROR --- status code --- 20:24:54 DEBUG --- status code --- 20:24:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:24:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:54 ERROR --- http response --- 20:24:54 DEBUG --- http response --- 20:24:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:24:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:24:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641096" 20:24:56 INFO [http_cmd]: http status code OK 20:24:56 DEBUG --- status code --- 20:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:56 DEBUG --- http response --- 20:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:24:56 DEBUG ----- output ----- 20:24:56 DEBUG { 20:24:56 DEBUG "status": "success", 20:24:56 DEBUG "data": { 20:24:56 DEBUG "resultType": "vector", 20:24:56 DEBUG "result": [ 20:24:56 DEBUG { 20:24:56 DEBUG "metric": { 20:24:56 DEBUG "pod": "ds-idrepo-0" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG [...] 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG "pod": "ds-idrepo-2" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 1713641096, 20:24:56 DEBUG "0" 20:24:56 DEBUG ] 20:24:56 DEBUG } 20:24:56 DEBUG ] 20:24:56 DEBUG } 20:24:56 DEBUG } 20:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641096" 20:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641096" 20:24:56 INFO [http_cmd]: http status code OK 20:24:56 DEBUG --- status code --- 20:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:56 DEBUG --- http response --- 20:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:24:56 DEBUG ----- output ----- 20:24:56 DEBUG { 20:24:56 DEBUG "status": "success", 20:24:56 DEBUG "data": { 20:24:56 DEBUG "resultType": "vector", 20:24:56 DEBUG "result": [ 20:24:56 DEBUG { 20:24:56 DEBUG "metric": { 20:24:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG [...] 20:24:56 DEBUG 20:24:56 INFO [http_cmd]: http status code OK 20:24:56 DEBUG 20:24:56 DEBUG --- status code --- 20:24:56 DEBUG 20:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:24:56 DEBUG }, 20:24:56 DEBUG --- http response --- 20:24:56 DEBUG "value": [ 20:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:24:56 DEBUG 1713641096, 20:24:56 DEBUG ----- output ----- 20:24:56 DEBUG "2840.6666666666665" 20:24:56 DEBUG { 20:24:56 DEBUG ] 20:24:56 DEBUG "status": "success", 20:24:56 DEBUG } 20:24:56 DEBUG "data": { 20:24:56 DEBUG ] 20:24:56 DEBUG "resultType": "vector", 20:24:56 DEBUG } 20:24:56 DEBUG "result": [ 20:24:56 DEBUG } 20:24:56 DEBUG { 20:24:56 DEBUG "metric": { 20:24:56 DEBUG "pod": "ds-idrepo-0" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG [...] 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG "pod": "ds-idrepo-2" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 1713641096, 20:24:56 DEBUG "0" 20:24:56 DEBUG ] 20:24:56 DEBUG } 20:24:56 DEBUG ] 20:24:56 DEBUG } 20:24:56 DEBUG } 20:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641096" 20:24:56 INFO [http_cmd]: http status code OK 20:24:56 DEBUG --- status code --- 20:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:56 DEBUG --- http response --- 20:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:24:56 DEBUG ----- output ----- 20:24:56 DEBUG { 20:24:56 DEBUG "status": "success", 20:24:56 DEBUG "data": { 20:24:56 DEBUG "resultType": "vector", 20:24:56 DEBUG "result": [ 20:24:56 DEBUG { 20:24:56 DEBUG "metric": { 20:24:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG [...] 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 1713641096, 20:24:56 DEBUG "0" 20:24:56 DEBUG ] 20:24:56 DEBUG } 20:24:56 DEBUG ] 20:24:56 DEBUG } 20:24:56 DEBUG } 20:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641096" 20:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713641096" 20:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641096" 20:24:56 INFO [http_cmd]: http status code OK 20:24:56 DEBUG --- status code --- 20:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:56 DEBUG --- http response --- 20:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:24:56 DEBUG ----- output ----- 20:24:56 DEBUG { 20:24:56 DEBUG "status": "success", 20:24:56 DEBUG "data": { 20:24:56 DEBUG "resultType": "vector", 20:24:56 DEBUG "result": [ 20:24:56 DEBUG { 20:24:56 DEBUG "metric": { 20:24:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG [...] 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 1713641096, 20:24:56 DEBUG "2823.3" 20:24:56 DEBUG ] 20:24:56 DEBUG } 20:24:56 DEBUG ] 20:24:56 DEBUG } 20:24:56 DEBUG } 20:24:56 INFO [http_cmd]: http status code OK 20:24:56 DEBUG --- status code --- 20:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:56 DEBUG --- http response --- 20:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:24:56 DEBUG ----- output ----- 20:24:56 DEBUG { 20:24:56 DEBUG "status": "success", 20:24:56 DEBUG "data": { 20:24:56 DEBUG "resultType": "vector", 20:24:56 DEBUG "result": [ 20:24:56 DEBUG { 20:24:56 DEBUG "metric": { 20:24:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG [...] 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 1713641096, 20:24:56 DEBUG "9.541200794336817" 20:24:56 DEBUG ] 20:24:56 DEBUG } 20:24:56 DEBUG ] 20:24:56 DEBUG } 20:24:56 DEBUG } 20:24:56 INFO [http_cmd]: http status code OK 20:24:56 DEBUG --- status code --- 20:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:56 DEBUG --- http response --- 20:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:24:56 DEBUG ----- output ----- 20:24:56 DEBUG { 20:24:56 DEBUG "status": "success", 20:24:56 DEBUG "data": { 20:24:56 DEBUG "resultType": "vector", 20:24:56 DEBUG "result": [ 20:24:56 DEBUG { 20:24:56 DEBUG "metric": { 20:24:56 DEBUG "instance": "10.142.0.107:9100" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG [...] 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG 20:24:56 DEBUG "instance": "10.142.0.43:9100" 20:24:56 DEBUG }, 20:24:56 DEBUG "value": [ 20:24:56 DEBUG 1713641096, 20:24:56 DEBUG "3751702418432" 20:24:56 DEBUG ] 20:24:56 DEBUG } 20:24:56 DEBUG ] 20:24:56 DEBUG } 20:24:56 DEBUG } 20:24:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641097" 20:24:57 INFO [http_cmd]: http status code OK 20:24:57 DEBUG --- status code --- 20:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:57 DEBUG --- http response --- 20:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:24:57 DEBUG ----- output ----- 20:24:57 DEBUG { 20:24:57 DEBUG "status": "success", 20:24:57 DEBUG "data": { 20:24:57 DEBUG "resultType": "vector", 20:24:57 DEBUG "result": [ 20:24:57 DEBUG { 20:24:57 DEBUG "metric": { 20:24:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:24:57 DEBUG }, 20:24:57 DEBUG "value": [ 20:24:57 DEBUG 20:24:57 DEBUG 20:24:57 DEBUG 20:24:57 DEBUG [...] 20:24:57 DEBUG 20:24:57 DEBUG 20:24:57 DEBUG 20:24:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:24:57 DEBUG }, 20:24:57 DEBUG "value": [ 20:24:57 DEBUG 1713641097, 20:24:57 DEBUG "2822.7" 20:24:57 DEBUG ] 20:24:57 DEBUG } 20:24:57 DEBUG ] 20:24:57 DEBUG } 20:24:57 DEBUG } 20:24:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641097" 20:24:57 INFO [http_cmd]: http status code OK 20:24:57 DEBUG --- status code --- 20:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:57 DEBUG --- http response --- 20:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:24:57 DEBUG ----- output ----- 20:24:57 DEBUG { 20:24:57 DEBUG "status": "success", 20:24:57 DEBUG "data": { 20:24:57 DEBUG "resultType": "vector", 20:24:57 DEBUG "result": [ 20:24:57 DEBUG { 20:24:57 DEBUG "metric": { 20:24:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:24:57 DEBUG }, 20:24:57 DEBUG "value": [ 20:24:57 DEBUG 20:24:57 DEBUG 20:24:57 DEBUG 20:24:57 DEBUG [...] 20:24:57 DEBUG 20:24:57 DEBUG 20:24:57 DEBUG 20:24:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:24:57 DEBUG }, 20:24:57 DEBUG "value": [ 20:24:57 DEBUG 1713641097, 20:24:57 DEBUG "2823.1666666666665" 20:24:57 DEBUG ] 20:24:57 DEBUG } 20:24:57 DEBUG ] 20:24:57 DEBUG } 20:24:57 DEBUG } 20:24:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641097" 20:24:57 INFO [http_cmd]: http status code OK 20:24:57 DEBUG --- status code --- 20:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:57 DEBUG --- http response --- 20:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:24:57 DEBUG ----- output ----- 20:24:57 DEBUG { 20:24:57 DEBUG "status": "success", 20:24:57 DEBUG "data": { 20:24:57 DEBUG "resultType": "vector", 20:24:57 DEBUG "result": [ 20:24:57 DEBUG { 20:24:57 DEBUG "metric": { 20:24:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:24:57 DEBUG }, 20:24:57 DEBUG "value": [ 20:24:57 DEBUG 20:24:57 DEBUG 20:24:57 DEBUG 20:24:57 DEBUG [...] 20:24:57 DEBUG 20:24:57 DEBUG 20:24:57 DEBUG 20:24:57 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:24:57 DEBUG }, 20:24:57 DEBUG "value": [ 20:24:57 DEBUG 1713641097, 20:24:57 DEBUG "0" 20:24:57 DEBUG ] 20:24:57 DEBUG } 20:24:57 DEBUG ] 20:24:57 DEBUG } 20:24:57 DEBUG } 20:24:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641098" 20:24:58 INFO [http_cmd]: http status code OK 20:24:58 DEBUG --- status code --- 20:24:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:58 DEBUG --- http response --- 20:24:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:24:58 DEBUG ----- output ----- 20:24:58 DEBUG { 20:24:58 DEBUG "status": "success", 20:24:58 DEBUG "data": { 20:24:58 DEBUG "resultType": "vector", 20:24:58 DEBUG "result": [ 20:24:58 DEBUG { 20:24:58 DEBUG "metric": { 20:24:58 DEBUG "pod": "ds-idrepo-0" 20:24:58 DEBUG }, 20:24:58 DEBUG "value": [ 20:24:58 DEBUG 20:24:58 DEBUG 20:24:58 DEBUG 20:24:58 DEBUG [...] 20:24:58 DEBUG 20:24:58 DEBUG 20:24:58 DEBUG 20:24:58 DEBUG "pod": "ds-idrepo-2" 20:24:58 DEBUG }, 20:24:58 DEBUG "value": [ 20:24:58 DEBUG 1713641098, 20:24:58 DEBUG "0" 20:24:58 DEBUG ] 20:24:58 DEBUG } 20:24:58 DEBUG ] 20:24:58 DEBUG } 20:24:58 DEBUG } 20:24:58 INFO 20:24:58 INFO [loop_until]: kubectl --namespace=xlou top pods 20:24:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:24:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641098" 20:24:58 INFO [http_cmd]: http status code OK 20:24:58 DEBUG --- status code --- 20:24:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:58 DEBUG --- http response --- 20:24:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:24:58 DEBUG ----- output ----- 20:24:58 DEBUG { 20:24:58 DEBUG "status": "success", 20:24:58 DEBUG "data": { 20:24:58 DEBUG "resultType": "vector", 20:24:58 DEBUG "result": [ 20:24:58 DEBUG { 20:24:58 DEBUG "metric": { 20:24:58 DEBUG "pod": "ds-cts-0" 20:24:58 DEBUG }, 20:24:58 DEBUG "value": [ 20:24:58 DEBUG 20:24:58 DEBUG 20:24:58 DEBUG 20:24:58 DEBUG [...] 20:24:58 DEBUG 20:24:58 DEBUG 20:24:58 DEBUG 20:24:58 DEBUG "pod": "ds-cts-2" 20:24:58 DEBUG }, 20:24:58 DEBUG "value": [ 20:24:58 DEBUG 1713641098, 20:24:58 DEBUG "0" 20:24:58 DEBUG ] 20:24:58 DEBUG } 20:24:58 DEBUG ] 20:24:58 DEBUG } 20:24:58 DEBUG } 20:24:58 INFO [loop_until]: OK (rc = 0) 20:24:58 DEBUG --- stdout --- 20:24:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18241m 8662Mi am-767f6b67b5-qcqz2 18367m 9072Mi am-767f6b67b5-vbltz 20094m 8672Mi ds-cts-0 19769m 11041Mi ds-cts-1 18141m 10848Mi ds-cts-2 20038m 11096Mi ds-idrepo-0 7908m 3894Mi ds-idrepo-1 7674m 3885Mi ds-idrepo-2 8129m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9431m 847Mi 20:24:58 DEBUG --- stderr --- 20:24:58 DEBUG 20:24:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713641098" 20:24:58 INFO [http_cmd]: http status code OK 20:24:58 DEBUG --- status code --- 20:24:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:58 DEBUG --- http response --- 20:24:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:24:58 DEBUG ----- output ----- 20:24:58 DEBUG { 20:24:58 DEBUG "status": "success", 20:24:58 DEBUG "data": { 20:24:58 DEBUG "resultType": "vector", 20:24:58 DEBUG "result": [ 20:24:58 DEBUG { 20:24:58 DEBUG "metric": { 20:24:58 DEBUG "pod": "ds-idrepo-0" 20:24:58 DEBUG }, 20:24:58 DEBUG "value": [ 20:24:58 DEBUG 20:24:58 DEBUG 20:24:58 DEBUG 20:24:58 DEBUG [...] 20:24:58 DEBUG 20:24:58 DEBUG 20:24:58 DEBUG 20:24:58 DEBUG "pod": "ds-cts-2" 20:24:58 DEBUG }, 20:24:58 DEBUG "value": [ 20:24:58 DEBUG 1713641098, 20:24:58 DEBUG "0" 20:24:58 DEBUG ] 20:24:58 DEBUG } 20:24:58 DEBUG ] 20:24:58 DEBUG } 20:24:58 DEBUG } 20:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641099" 20:24:59 INFO [http_cmd]: http status code OK 20:24:59 DEBUG --- status code --- 20:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:59 DEBUG --- http response --- 20:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:24:59 DEBUG ----- output ----- 20:24:59 DEBUG { 20:24:59 DEBUG "status": "success", 20:24:59 DEBUG "data": { 20:24:59 DEBUG "resultType": "vector", 20:24:59 DEBUG "result": [ 20:24:59 DEBUG { 20:24:59 DEBUG "metric": { 20:24:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:24:59 DEBUG }, 20:24:59 DEBUG "value": [ 20:24:59 DEBUG 20:24:59 DEBUG 20:24:59 DEBUG 20:24:59 DEBUG [...] 20:24:59 DEBUG 20:24:59 DEBUG 20:24:59 DEBUG 20:24:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:24:59 DEBUG }, 20:24:59 DEBUG "value": [ 20:24:59 DEBUG 1713641099, 20:24:59 DEBUG "509766" 20:24:59 DEBUG ] 20:24:59 DEBUG } 20:24:59 DEBUG ] 20:24:59 DEBUG } 20:24:59 DEBUG } 20:24:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641099" 20:24:59 INFO [http_cmd]: http status code OK 20:24:59 DEBUG --- status code --- 20:24:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:24:59 DEBUG --- http response --- 20:24:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:24:59 DEBUG ----- output ----- 20:24:59 DEBUG { 20:24:59 DEBUG "status": "success", 20:24:59 DEBUG "data": { 20:24:59 DEBUG "resultType": "vector", 20:24:59 DEBUG "result": [ 20:24:59 DEBUG { 20:24:59 DEBUG "metric": { 20:24:59 DEBUG "instance": "10.142.0.107:9100" 20:24:59 DEBUG }, 20:24:59 DEBUG "value": [ 20:24:59 DEBUG 20:24:59 DEBUG 20:24:59 DEBUG 20:24:59 DEBUG [...] 20:24:59 DEBUG 20:24:59 DEBUG 20:24:59 DEBUG 20:24:59 DEBUG "instance": "10.142.0.43:9100" 20:24:59 DEBUG }, 20:24:59 DEBUG "value": [ 20:24:59 DEBUG 1713641099, 20:24:59 DEBUG "0.17213333333326752" 20:24:59 DEBUG ] 20:24:59 DEBUG } 20:24:59 DEBUG ] 20:24:59 DEBUG } 20:24:59 DEBUG } 20:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641100" 20:25:00 INFO [http_cmd]: http status code OK 20:25:00 DEBUG --- status code --- 20:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:00 DEBUG --- http response --- 20:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:25:00 DEBUG ----- output ----- 20:25:00 DEBUG { 20:25:00 DEBUG "status": "success", 20:25:00 DEBUG "data": { 20:25:00 DEBUG "resultType": "vector", 20:25:00 DEBUG "result": [ 20:25:00 DEBUG { 20:25:00 DEBUG "metric": { 20:25:00 DEBUG "instance": "10.142.0.107:9100" 20:25:00 DEBUG }, 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG [...] 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG "instance": "10.142.0.43:9100" 20:25:00 DEBUG }, 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 1713641100, 20:25:00 DEBUG "85522351.26666667" 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713641100" 20:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641100" 20:25:00 INFO [http_cmd]: http status code OK 20:25:00 DEBUG --- status code --- 20:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:00 DEBUG --- http response --- 20:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:25:00 DEBUG ----- output ----- 20:25:00 DEBUG { 20:25:00 DEBUG "status": "success", 20:25:00 DEBUG "data": { 20:25:00 DEBUG "resultType": "vector", 20:25:00 DEBUG "result": [ 20:25:00 DEBUG { 20:25:00 DEBUG "metric": { 20:25:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:25:00 DEBUG }, 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG [...] 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:25:00 DEBUG }, 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 1713641100, 20:25:00 DEBUG "984215552" 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 INFO [http_cmd]: http status code OK 20:25:00 DEBUG --- status code --- 20:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:00 DEBUG --- http response --- 20:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:25:00 DEBUG ----- output ----- 20:25:00 DEBUG { 20:25:00 DEBUG "status": "success", 20:25:00 DEBUG "data": { 20:25:00 DEBUG "resultType": "vector", 20:25:00 DEBUG "result": [ 20:25:00 DEBUG { 20:25:00 DEBUG "metric": { 20:25:00 DEBUG "instance": "10.142.0.107:9100" 20:25:00 DEBUG }, 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG [...] 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG "instance": "10.142.0.43:9100" 20:25:00 DEBUG }, 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 1713641100, 20:25:00 DEBUG "85526005.33333333" 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641100" 20:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641100" 20:25:00 INFO [http_cmd]: http status code OK 20:25:00 DEBUG --- status code --- 20:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:00 DEBUG --- http response --- 20:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:25:00 DEBUG ----- output ----- 20:25:00 DEBUG { 20:25:00 DEBUG "status": "success", 20:25:00 DEBUG "data": { 20:25:00 DEBUG "resultType": "vector", 20:25:00 DEBUG "result": [ 20:25:00 DEBUG { 20:25:00 DEBUG "metric": { 20:25:00 DEBUG "pod": "ds-idrepo-1" 20:25:00 DEBUG }, 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG [...] 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:25:00 DEBUG }, 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 1713641100, 20:25:00 DEBUG "0" 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 INFO [http_cmd]: http status code OK 20:25:00 DEBUG --- status code --- 20:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:00 DEBUG --- http response --- 20:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:25:00 DEBUG ----- output ----- 20:25:00 DEBUG { 20:25:00 DEBUG "status": "success", 20:25:00 DEBUG "data": { 20:25:00 DEBUG "resultType": "vector", 20:25:00 DEBUG "result": [ 20:25:00 DEBUG { 20:25:00 DEBUG "metric": { 20:25:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:25:00 DEBUG }, 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG [...] 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:25:00 DEBUG }, 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 1713641100, 20:25:00 DEBUG "0" 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641100" 20:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713641100" 20:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641100" 20:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641100" 20:25:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641100" 20:25:00 INFO [http_cmd]: http status code OK 20:25:00 DEBUG --- status code --- 20:25:00 INFO [http_cmd]: http status code OK 20:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:00 INFO [http_cmd]: http status code OK 20:25:00 DEBUG --- status code --- 20:25:00 DEBUG --- http response --- 20:25:00 DEBUG --- status code --- 20:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:25:00 DEBUG --- http response --- 20:25:00 DEBUG --- http response --- 20:25:00 DEBUG ----- output ----- 20:25:00 INFO [http_cmd]: http status code OK 20:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:25:00 DEBUG { 20:25:00 DEBUG --- status code --- 20:25:00 DEBUG ----- output ----- 20:25:00 DEBUG ----- output ----- 20:25:00 DEBUG { 20:25:00 DEBUG "status": "success", 20:25:00 DEBUG { 20:25:00 DEBUG "data": { 20:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:00 DEBUG "status": "success", 20:25:00 DEBUG "status": "success", 20:25:00 DEBUG "data": { 20:25:00 DEBUG --- http response --- 20:25:00 DEBUG "data": { 20:25:00 DEBUG "resultType": "vector", 20:25:00 DEBUG "resultType": "vector", 20:25:00 DEBUG "resultType": "vector", 20:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:25:00 INFO [http_cmd]: http status code OK 20:25:00 DEBUG "result": [ 20:25:00 DEBUG "result": [ 20:25:00 DEBUG "result": [ 20:25:00 DEBUG ----- output ----- 20:25:00 DEBUG --- status code --- 20:25:00 DEBUG { 20:25:00 DEBUG { 20:25:00 DEBUG { 20:25:00 DEBUG { 20:25:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:00 DEBUG "metric": { 20:25:00 DEBUG "metric": { 20:25:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:25:00 DEBUG "status": "success", 20:25:00 DEBUG --- http response --- 20:25:00 DEBUG "pod": "ds-idrepo-1" 20:25:00 DEBUG }, 20:25:00 DEBUG "metric": { 20:25:00 DEBUG "data": { 20:25:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:25:00 DEBUG }, 20:25:00 DEBUG "value": [ 20:25:00 DEBUG "pod": "ds-cts-0" 20:25:00 DEBUG "resultType": "vector", 20:25:00 DEBUG ----- output ----- 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 20:25:00 DEBUG }, 20:25:00 DEBUG "result": [ 20:25:00 DEBUG { 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG "value": [ 20:25:00 DEBUG { 20:25:00 DEBUG "status": "success", 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG "metric": { 20:25:00 DEBUG "data": { 20:25:00 DEBUG [...] 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG "pod": "ds-cts-0" 20:25:00 DEBUG "resultType": "vector", 20:25:00 DEBUG 20:25:00 DEBUG [...] 20:25:00 DEBUG 20:25:00 DEBUG }, 20:25:00 DEBUG "result": [ 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG [...] 20:25:00 DEBUG 20:25:00 DEBUG { 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG "metric": { 20:25:00 DEBUG "value": [ 20:25:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG "instance": "10.142.0.107:9100" 20:25:00 DEBUG 20:25:00 DEBUG }, 20:25:00 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:25:00 DEBUG }, 20:25:00 DEBUG }, 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG [...] 20:25:00 DEBUG "value": [ 20:25:00 DEBUG "pod": "ds-cts-2" 20:25:00 DEBUG 20:25:00 DEBUG 1713641100, 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 20:25:00 DEBUG 20:25:00 DEBUG [...] 20:25:00 DEBUG }, 20:25:00 DEBUG 1713641100, 20:25:00 DEBUG "0" 20:25:00 DEBUG "instance": "10.142.0.43:9100" 20:25:00 DEBUG 20:25:00 DEBUG "value": [ 20:25:00 DEBUG "0.159" 20:25:00 DEBUG ] 20:25:00 DEBUG }, 20:25:00 DEBUG 20:25:00 DEBUG 1713641100, 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG "value": [ 20:25:00 DEBUG 20:25:00 DEBUG "0.053602822222222214" 20:25:00 DEBUG } 20:25:00 DEBUG ] 20:25:00 DEBUG 1713641100, 20:25:00 DEBUG "pod": "ds-cts-2" 20:25:00 DEBUG ] 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG "0.05900000000000318" 20:25:00 DEBUG }, 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 DEBUG ] 20:25:00 DEBUG "value": [ 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 DEBUG 1713641100, 20:25:00 DEBUG "31.433333333333334" 20:25:00 DEBUG ] 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 DEBUG ] 20:25:00 DEBUG } 20:25:00 DEBUG } 20:25:00 INFO 20:25:00 INFO [loop_until]: kubectl --namespace=xlou top node 20:25:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:25:00 INFO [loop_until]: OK (rc = 0) 20:25:00 DEBUG --- stdout --- 20:25:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 60m 0% 6231Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7915m 26% 7644Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18229m 61% 11388Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3334Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19765m 66% 14236Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5187Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18483m 61% 11970Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15108m 50% 2976Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20004m 67% 10888Mi 9% gke-xlou-cdm-ds-0f601751-j488 8690m 29% 7229Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17974m 60% 15893Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8296m 27% 7851Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20007m 67% 13747Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9726m 61% 6396Mi 10% 20:25:00 DEBUG --- stderr --- 20:25:00 DEBUG 20:25:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713641154" 20:25:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641154" 20:25:54 ERROR [http_cmd]: ERROR 20:25:54 INFO [http_cmd]: http status code OK 20:25:54 ERROR --- status code --- 20:25:54 DEBUG --- status code --- 20:25:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:25:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:54 ERROR --- http response --- 20:25:54 DEBUG --- http response --- 20:25:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:25:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:25:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641156" 20:25:56 INFO [http_cmd]: http status code OK 20:25:56 DEBUG --- status code --- 20:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:56 DEBUG --- http response --- 20:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:25:56 DEBUG ----- output ----- 20:25:56 DEBUG { 20:25:56 DEBUG "status": "success", 20:25:56 DEBUG "data": { 20:25:56 DEBUG "resultType": "vector", 20:25:56 DEBUG "result": [ 20:25:56 DEBUG { 20:25:56 DEBUG "metric": { 20:25:56 DEBUG "pod": "ds-idrepo-0" 20:25:56 DEBUG }, 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG [...] 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG "pod": "ds-idrepo-2" 20:25:56 DEBUG }, 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 1713641156, 20:25:56 DEBUG "0" 20:25:56 DEBUG ] 20:25:56 DEBUG } 20:25:56 DEBUG ] 20:25:56 DEBUG } 20:25:56 DEBUG } 20:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641156" 20:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641156" 20:25:56 INFO [http_cmd]: http status code OK 20:25:56 INFO [http_cmd]: http status code OK 20:25:56 DEBUG --- status code --- 20:25:56 DEBUG --- status code --- 20:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:56 DEBUG --- http response --- 20:25:56 DEBUG --- http response --- 20:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:25:56 DEBUG ----- output ----- 20:25:56 DEBUG ----- output ----- 20:25:56 DEBUG { 20:25:56 DEBUG { 20:25:56 DEBUG "status": "success", 20:25:56 DEBUG "status": "success", 20:25:56 DEBUG "data": { 20:25:56 DEBUG "data": { 20:25:56 DEBUG "resultType": "vector", 20:25:56 DEBUG "resultType": "vector", 20:25:56 DEBUG "result": [ 20:25:56 DEBUG "result": [ 20:25:56 DEBUG { 20:25:56 DEBUG { 20:25:56 DEBUG "metric": { 20:25:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:25:56 DEBUG "metric": { 20:25:56 DEBUG }, 20:25:56 DEBUG "pod": "ds-idrepo-0" 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG [...] 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG }, 20:25:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:25:56 DEBUG "value": [ 20:25:56 DEBUG }, 20:25:56 DEBUG 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 20:25:56 DEBUG 1713641156, 20:25:56 DEBUG 20:25:56 DEBUG "2818.2" 20:25:56 DEBUG [...] 20:25:56 DEBUG ] 20:25:56 DEBUG 20:25:56 DEBUG } 20:25:56 DEBUG 20:25:56 DEBUG ] 20:25:56 DEBUG 20:25:56 DEBUG } 20:25:56 DEBUG "pod": "ds-idrepo-2" 20:25:56 DEBUG } 20:25:56 DEBUG }, 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 1713641156, 20:25:56 DEBUG "0" 20:25:56 DEBUG ] 20:25:56 DEBUG } 20:25:56 DEBUG ] 20:25:56 DEBUG } 20:25:56 DEBUG } 20:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641156" 20:25:56 INFO [http_cmd]: http status code OK 20:25:56 DEBUG --- status code --- 20:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:56 DEBUG --- http response --- 20:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:25:56 DEBUG ----- output ----- 20:25:56 DEBUG { 20:25:56 DEBUG "status": "success", 20:25:56 DEBUG "data": { 20:25:56 DEBUG "resultType": "vector", 20:25:56 DEBUG "result": [ 20:25:56 DEBUG { 20:25:56 DEBUG "metric": { 20:25:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:25:56 DEBUG }, 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG [...] 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:25:56 DEBUG }, 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 1713641156, 20:25:56 DEBUG "0" 20:25:56 DEBUG ] 20:25:56 DEBUG } 20:25:56 DEBUG ] 20:25:56 DEBUG } 20:25:56 DEBUG } 20:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641156" 20:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713641156" 20:25:56 INFO [http_cmd]: http status code OK 20:25:56 DEBUG --- status code --- 20:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:56 DEBUG --- http response --- 20:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:25:56 DEBUG ----- output ----- 20:25:56 DEBUG { 20:25:56 DEBUG "status": "success", 20:25:56 DEBUG "data": { 20:25:56 DEBUG "resultType": "vector", 20:25:56 DEBUG "result": [ 20:25:56 DEBUG { 20:25:56 DEBUG "metric": { 20:25:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:25:56 DEBUG }, 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG [...] 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:25:56 DEBUG }, 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 1713641156, 20:25:56 DEBUG "2827.0666666666666" 20:25:56 DEBUG ] 20:25:56 DEBUG } 20:25:56 DEBUG ] 20:25:56 DEBUG } 20:25:56 DEBUG } 20:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641156" 20:25:56 INFO [http_cmd]: http status code OK 20:25:56 DEBUG --- status code --- 20:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:56 DEBUG --- http response --- 20:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:25:56 DEBUG ----- output ----- 20:25:56 DEBUG { 20:25:56 DEBUG "status": "success", 20:25:56 DEBUG "data": { 20:25:56 DEBUG "resultType": "vector", 20:25:56 DEBUG "result": [ 20:25:56 DEBUG { 20:25:56 DEBUG "metric": { 20:25:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:25:56 DEBUG }, 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG [...] 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:25:56 DEBUG }, 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 1713641156, 20:25:56 DEBUG "9.54571317221876" 20:25:56 DEBUG ] 20:25:56 DEBUG } 20:25:56 DEBUG ] 20:25:56 DEBUG } 20:25:56 DEBUG } 20:25:56 INFO [http_cmd]: http status code OK 20:25:56 DEBUG --- status code --- 20:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:56 DEBUG --- http response --- 20:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:25:56 DEBUG ----- output ----- 20:25:56 DEBUG { 20:25:56 DEBUG "status": "success", 20:25:56 DEBUG "data": { 20:25:56 DEBUG "resultType": "vector", 20:25:56 DEBUG "result": [ 20:25:56 DEBUG { 20:25:56 DEBUG "metric": { 20:25:56 DEBUG "instance": "10.142.0.107:9100" 20:25:56 DEBUG }, 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG [...] 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG 20:25:56 DEBUG "instance": "10.142.0.43:9100" 20:25:56 DEBUG }, 20:25:56 DEBUG "value": [ 20:25:56 DEBUG 1713641156, 20:25:56 DEBUG "3755661835264" 20:25:56 DEBUG ] 20:25:56 DEBUG } 20:25:56 DEBUG ] 20:25:56 DEBUG } 20:25:56 DEBUG } 20:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641157" 20:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641157" 20:25:57 INFO [http_cmd]: http status code OK 20:25:57 DEBUG --- status code --- 20:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:57 DEBUG --- http response --- 20:25:57 INFO [http_cmd]: http status code OK 20:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:25:57 DEBUG --- status code --- 20:25:57 DEBUG ----- output ----- 20:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:57 DEBUG { 20:25:57 DEBUG --- http response --- 20:25:57 DEBUG "status": "success", 20:25:57 DEBUG "data": { 20:25:57 DEBUG "resultType": "vector", 20:25:57 DEBUG "result": [ 20:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:25:57 DEBUG { 20:25:57 DEBUG ----- output ----- 20:25:57 DEBUG "metric": { 20:25:57 DEBUG { 20:25:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:25:57 DEBUG "status": "success", 20:25:57 DEBUG }, 20:25:57 DEBUG "data": { 20:25:57 DEBUG "value": [ 20:25:57 DEBUG "resultType": "vector", 20:25:57 DEBUG 20:25:57 DEBUG "result": [ 20:25:57 DEBUG 20:25:57 DEBUG { 20:25:57 DEBUG 20:25:57 DEBUG "metric": { 20:25:57 DEBUG [...] 20:25:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:25:57 DEBUG 20:25:57 DEBUG }, 20:25:57 DEBUG 20:25:57 DEBUG 20:25:57 DEBUG "value": [ 20:25:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:25:57 DEBUG 20:25:57 DEBUG }, 20:25:57 DEBUG 20:25:57 DEBUG "value": [ 20:25:57 DEBUG 20:25:57 DEBUG 1713641157, 20:25:57 DEBUG [...] 20:25:57 DEBUG "2827.633333333333" 20:25:57 DEBUG 20:25:57 DEBUG ] 20:25:57 DEBUG 20:25:57 DEBUG } 20:25:57 DEBUG 20:25:57 DEBUG ] 20:25:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:25:57 DEBUG } 20:25:57 DEBUG }, 20:25:57 DEBUG } 20:25:57 DEBUG "value": [ 20:25:57 DEBUG 1713641157, 20:25:57 DEBUG "2827.9" 20:25:57 DEBUG ] 20:25:57 DEBUG } 20:25:57 DEBUG ] 20:25:57 DEBUG } 20:25:57 DEBUG } 20:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641157" 20:25:57 INFO [http_cmd]: http status code OK 20:25:57 DEBUG --- status code --- 20:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:57 DEBUG --- http response --- 20:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:25:57 DEBUG ----- output ----- 20:25:57 DEBUG { 20:25:57 DEBUG "status": "success", 20:25:57 DEBUG "data": { 20:25:57 DEBUG "resultType": "vector", 20:25:57 DEBUG "result": [ 20:25:57 DEBUG { 20:25:57 DEBUG "metric": { 20:25:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:25:57 DEBUG }, 20:25:57 DEBUG "value": [ 20:25:57 DEBUG 20:25:57 DEBUG 20:25:57 DEBUG 20:25:57 DEBUG [...] 20:25:57 DEBUG 20:25:57 DEBUG 20:25:57 DEBUG 20:25:57 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:25:57 DEBUG }, 20:25:57 DEBUG "value": [ 20:25:57 DEBUG 1713641157, 20:25:57 DEBUG "9.49758606467364" 20:25:57 DEBUG ] 20:25:57 DEBUG } 20:25:57 DEBUG ] 20:25:57 DEBUG } 20:25:57 DEBUG } 20:25:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641158" 20:25:58 INFO [http_cmd]: http status code OK 20:25:58 DEBUG --- status code --- 20:25:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:58 DEBUG --- http response --- 20:25:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:25:58 DEBUG ----- output ----- 20:25:58 DEBUG { 20:25:58 DEBUG "status": "success", 20:25:58 DEBUG "data": { 20:25:58 DEBUG "resultType": "vector", 20:25:58 DEBUG "result": [ 20:25:58 DEBUG { 20:25:58 DEBUG "metric": { 20:25:58 DEBUG "pod": "ds-idrepo-0" 20:25:58 DEBUG }, 20:25:58 DEBUG "value": [ 20:25:58 DEBUG 20:25:58 DEBUG 20:25:58 DEBUG 20:25:58 DEBUG [...] 20:25:58 DEBUG 20:25:58 DEBUG 20:25:58 DEBUG 20:25:58 DEBUG "pod": "ds-idrepo-2" 20:25:58 DEBUG }, 20:25:58 DEBUG "value": [ 20:25:58 DEBUG 1713641158, 20:25:58 DEBUG "0" 20:25:58 DEBUG ] 20:25:58 DEBUG } 20:25:58 DEBUG ] 20:25:58 DEBUG } 20:25:58 DEBUG } 20:25:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641158" 20:25:58 INFO 20:25:58 INFO [loop_until]: kubectl --namespace=xlou top pods 20:25:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:25:58 INFO [http_cmd]: http status code OK 20:25:58 DEBUG --- status code --- 20:25:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:58 DEBUG --- http response --- 20:25:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:25:58 DEBUG ----- output ----- 20:25:58 DEBUG { 20:25:58 DEBUG "status": "success", 20:25:58 DEBUG "data": { 20:25:58 DEBUG "resultType": "vector", 20:25:58 DEBUG "result": [ 20:25:58 DEBUG { 20:25:58 DEBUG "metric": { 20:25:58 DEBUG "pod": "ds-cts-0" 20:25:58 DEBUG }, 20:25:58 DEBUG "value": [ 20:25:58 DEBUG 20:25:58 DEBUG 20:25:58 DEBUG 20:25:58 DEBUG [...] 20:25:58 DEBUG 20:25:58 DEBUG 20:25:58 DEBUG 20:25:58 DEBUG "pod": "ds-cts-2" 20:25:58 DEBUG }, 20:25:58 DEBUG "value": [ 20:25:58 DEBUG 1713641158, 20:25:58 DEBUG "0" 20:25:58 DEBUG ] 20:25:58 DEBUG } 20:25:58 DEBUG ] 20:25:58 DEBUG } 20:25:58 DEBUG } 20:25:58 INFO [loop_until]: OK (rc = 0) 20:25:58 DEBUG --- stdout --- 20:25:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18451m 8660Mi am-767f6b67b5-qcqz2 18506m 9073Mi am-767f6b67b5-vbltz 19597m 8670Mi ds-cts-0 19302m 11046Mi ds-cts-1 18848m 10835Mi ds-cts-2 19826m 11102Mi ds-idrepo-0 7949m 3894Mi ds-idrepo-1 7966m 3886Mi ds-idrepo-2 8042m 3829Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 3m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 9269m 849Mi 20:25:58 DEBUG --- stderr --- 20:25:58 DEBUG 20:25:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713641158" 20:25:58 INFO [http_cmd]: http status code OK 20:25:58 DEBUG --- status code --- 20:25:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:58 DEBUG --- http response --- 20:25:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:25:58 DEBUG ----- output ----- 20:25:58 DEBUG { 20:25:58 DEBUG "status": "success", 20:25:58 DEBUG "data": { 20:25:58 DEBUG "resultType": "vector", 20:25:58 DEBUG "result": [ 20:25:58 DEBUG { 20:25:58 DEBUG "metric": { 20:25:58 DEBUG "pod": "ds-idrepo-0" 20:25:58 DEBUG }, 20:25:58 DEBUG "value": [ 20:25:58 DEBUG 20:25:58 DEBUG 20:25:58 DEBUG 20:25:58 DEBUG [...] 20:25:58 DEBUG 20:25:58 DEBUG 20:25:58 DEBUG 20:25:58 DEBUG "pod": "ds-cts-2" 20:25:58 DEBUG }, 20:25:58 DEBUG "value": [ 20:25:58 DEBUG 1713641158, 20:25:58 DEBUG "0" 20:25:58 DEBUG ] 20:25:58 DEBUG } 20:25:58 DEBUG ] 20:25:58 DEBUG } 20:25:58 DEBUG } 20:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641159" 20:25:59 INFO [http_cmd]: http status code OK 20:25:59 DEBUG --- status code --- 20:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:59 DEBUG --- http response --- 20:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:25:59 DEBUG ----- output ----- 20:25:59 DEBUG { 20:25:59 DEBUG "status": "success", 20:25:59 DEBUG "data": { 20:25:59 DEBUG "resultType": "vector", 20:25:59 DEBUG "result": [ 20:25:59 DEBUG { 20:25:59 DEBUG "metric": { 20:25:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:25:59 DEBUG }, 20:25:59 DEBUG "value": [ 20:25:59 DEBUG 20:25:59 DEBUG 20:25:59 DEBUG 20:25:59 DEBUG [...] 20:25:59 DEBUG 20:25:59 DEBUG 20:25:59 DEBUG 20:25:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:25:59 DEBUG }, 20:25:59 DEBUG "value": [ 20:25:59 DEBUG 1713641159, 20:25:59 DEBUG "509640" 20:25:59 DEBUG ] 20:25:59 DEBUG } 20:25:59 DEBUG ] 20:25:59 DEBUG } 20:25:59 DEBUG } 20:25:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641159" 20:25:59 INFO [http_cmd]: http status code OK 20:25:59 DEBUG --- status code --- 20:25:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:25:59 DEBUG --- http response --- 20:25:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:25:59 DEBUG ----- output ----- 20:25:59 DEBUG { 20:25:59 DEBUG "status": "success", 20:25:59 DEBUG "data": { 20:25:59 DEBUG "resultType": "vector", 20:25:59 DEBUG "result": [ 20:25:59 DEBUG { 20:25:59 DEBUG "metric": { 20:25:59 DEBUG "instance": "10.142.0.107:9100" 20:25:59 DEBUG }, 20:25:59 DEBUG "value": [ 20:25:59 DEBUG 20:25:59 DEBUG 20:25:59 DEBUG 20:25:59 DEBUG [...] 20:25:59 DEBUG 20:25:59 DEBUG 20:25:59 DEBUG 20:25:59 DEBUG "instance": "10.142.0.43:9100" 20:25:59 DEBUG }, 20:25:59 DEBUG "value": [ 20:25:59 DEBUG 1713641159, 20:25:59 DEBUG "0.29483333333337786" 20:25:59 DEBUG ] 20:25:59 DEBUG } 20:25:59 DEBUG ] 20:25:59 DEBUG } 20:25:59 DEBUG } 20:26:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641160" 20:26:00 INFO [http_cmd]: http status code OK 20:26:00 DEBUG --- status code --- 20:26:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:00 DEBUG --- http response --- 20:26:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:26:00 DEBUG ----- output ----- 20:26:00 DEBUG { 20:26:00 DEBUG "status": "success", 20:26:00 DEBUG "data": { 20:26:00 DEBUG "resultType": "vector", 20:26:00 DEBUG "result": [ 20:26:00 DEBUG { 20:26:00 DEBUG "metric": { 20:26:00 DEBUG "instance": "10.142.0.107:9100" 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG [...] 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG "instance": "10.142.0.43:9100" 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 1713641160, 20:26:00 DEBUG "85546079.1" 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG } 20:26:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713641160" 20:26:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641160" 20:26:00 INFO [http_cmd]: http status code OK 20:26:00 DEBUG --- status code --- 20:26:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:00 DEBUG --- http response --- 20:26:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:26:00 DEBUG ----- output ----- 20:26:00 DEBUG { 20:26:00 DEBUG "status": "success", 20:26:00 DEBUG "data": { 20:26:00 DEBUG "resultType": "vector", 20:26:00 DEBUG "result": [ 20:26:00 DEBUG { 20:26:00 DEBUG "metric": { 20:26:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG [...] 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 1713641160, 20:26:00 DEBUG "987332608" 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG } 20:26:00 INFO [http_cmd]: http status code OK 20:26:00 DEBUG --- status code --- 20:26:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:00 DEBUG --- http response --- 20:26:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:26:00 DEBUG ----- output ----- 20:26:00 DEBUG { 20:26:00 DEBUG "status": "success", 20:26:00 DEBUG "data": { 20:26:00 DEBUG "resultType": "vector", 20:26:00 DEBUG "result": [ 20:26:00 DEBUG { 20:26:00 DEBUG "metric": { 20:26:00 DEBUG "instance": "10.142.0.107:9100" 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG [...] 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG "instance": "10.142.0.43:9100" 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 1713641160, 20:26:00 DEBUG "85548891.9" 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG } 20:26:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641160" 20:26:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641160" 20:26:00 INFO [http_cmd]: http status code OK 20:26:00 DEBUG --- status code --- 20:26:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:00 DEBUG --- http response --- 20:26:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:26:00 DEBUG ----- output ----- 20:26:00 DEBUG { 20:26:00 DEBUG "status": "success", 20:26:00 DEBUG "data": { 20:26:00 DEBUG "resultType": "vector", 20:26:00 DEBUG "result": [ 20:26:00 DEBUG { 20:26:00 DEBUG "metric": { 20:26:00 DEBUG "pod": "ds-idrepo-0" 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG [...] 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 1713641160, 20:26:00 DEBUG "37001.50968870493" 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG } 20:26:00 INFO [http_cmd]: http status code OK 20:26:00 DEBUG --- status code --- 20:26:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:00 DEBUG --- http response --- 20:26:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:26:00 DEBUG ----- output ----- 20:26:00 DEBUG { 20:26:00 DEBUG "status": "success", 20:26:00 DEBUG "data": { 20:26:00 DEBUG "resultType": "vector", 20:26:00 DEBUG "result": [ 20:26:00 DEBUG { 20:26:00 DEBUG "metric": { 20:26:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG [...] 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 1713641160, 20:26:00 DEBUG "0" 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG } 20:26:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641160" 20:26:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713641160" 20:26:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641160" 20:26:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641160" 20:26:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641160" 20:26:00 INFO [http_cmd]: http status code OK 20:26:00 INFO [http_cmd]: http status code OK 20:26:00 DEBUG --- status code --- 20:26:00 DEBUG --- status code --- 20:26:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:00 DEBUG --- http response --- 20:26:00 DEBUG --- http response --- 20:26:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:26:00 INFO [http_cmd]: http status code OK 20:26:00 DEBUG ----- output ----- 20:26:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:26:00 DEBUG --- status code --- 20:26:00 DEBUG { 20:26:00 DEBUG "status": "success", 20:26:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:00 DEBUG ----- output ----- 20:26:00 DEBUG "data": { 20:26:00 INFO [http_cmd]: http status code OK 20:26:00 DEBUG --- http response --- 20:26:00 DEBUG { 20:26:00 DEBUG "resultType": "vector", 20:26:00 DEBUG --- status code --- 20:26:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:26:00 DEBUG "status": "success", 20:26:00 DEBUG "result": [ 20:26:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:00 DEBUG ----- output ----- 20:26:00 DEBUG { 20:26:00 DEBUG { 20:26:00 DEBUG --- http response --- 20:26:00 DEBUG "status": "success", 20:26:00 DEBUG "data": { 20:26:00 DEBUG "metric": { 20:26:00 DEBUG "pod": "ds-cts-0" 20:26:00 DEBUG "data": { 20:26:00 INFO [http_cmd]: http status code OK 20:26:00 DEBUG "resultType": "vector", 20:26:00 DEBUG }, 20:26:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:26:00 DEBUG ----- output ----- 20:26:00 DEBUG --- status code --- 20:26:00 DEBUG "result": [ 20:26:00 DEBUG "value": [ 20:26:00 DEBUG { 20:26:00 DEBUG "resultType": "vector", 20:26:00 DEBUG "result": [ 20:26:00 DEBUG { 20:26:00 DEBUG 20:26:00 DEBUG "status": "success", 20:26:00 DEBUG "metric": { 20:26:00 DEBUG { 20:26:00 DEBUG 20:26:00 DEBUG "data": { 20:26:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:26:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:00 DEBUG "metric": { 20:26:00 DEBUG 20:26:00 DEBUG "resultType": "vector", 20:26:00 DEBUG }, 20:26:00 DEBUG --- http response --- 20:26:00 DEBUG "pod": "ds-idrepo-0" 20:26:00 DEBUG [...] 20:26:00 DEBUG "result": [ 20:26:00 DEBUG "value": [ 20:26:00 DEBUG }, 20:26:00 DEBUG 20:26:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:26:00 DEBUG { 20:26:00 DEBUG 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 20:26:00 DEBUG ----- output ----- 20:26:00 DEBUG "metric": { 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG { 20:26:00 DEBUG "instance": "10.142.0.107:9100" 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 20:26:00 DEBUG "status": "success", 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG "pod": "ds-cts-2" 20:26:00 DEBUG "data": { 20:26:00 DEBUG 20:26:00 DEBUG [...] 20:26:00 DEBUG [...] 20:26:00 DEBUG }, 20:26:00 DEBUG "resultType": "vector", 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG "value": [ 20:26:00 DEBUG "result": [ 20:26:00 DEBUG [...] 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 1713641160, 20:26:00 DEBUG { 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG "31.9" 20:26:00 DEBUG "metric": { 20:26:00 DEBUG 20:26:00 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:26:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:26:00 DEBUG ] 20:26:00 DEBUG "pod": "ds-cts-0" 20:26:00 DEBUG 20:26:00 DEBUG "instance": "10.142.0.43:9100" 20:26:00 DEBUG }, 20:26:00 DEBUG } 20:26:00 DEBUG }, 20:26:00 DEBUG }, 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG ] 20:26:00 DEBUG "value": [ 20:26:00 DEBUG "value": [ 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 1713641160, 20:26:00 DEBUG } 20:26:00 DEBUG 20:26:00 DEBUG 1713641160, 20:26:00 DEBUG 1713641160, 20:26:00 DEBUG "0.159" 20:26:00 DEBUG } 20:26:00 DEBUG 20:26:00 DEBUG 20:26:00 DEBUG "0" 20:26:00 DEBUG ] 20:26:00 DEBUG [...] 20:26:00 DEBUG "0.10433333333332938" 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG } 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG 20:26:00 DEBUG } 20:26:00 DEBUG } 20:26:00 DEBUG "pod": "ds-cts-2" 20:26:00 DEBUG } 20:26:00 DEBUG }, 20:26:00 DEBUG "value": [ 20:26:00 DEBUG 1713641160, 20:26:00 DEBUG "0.021733333333333323" 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG ] 20:26:00 DEBUG } 20:26:00 DEBUG } 20:26:00 INFO 20:26:00 INFO [loop_until]: kubectl --namespace=xlou top node 20:26:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:26:00 INFO [loop_until]: OK (rc = 0) 20:26:00 DEBUG --- stdout --- 20:26:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6222Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8132m 27% 7642Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 18299m 61% 11371Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3340Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19545m 65% 14233Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5191Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18457m 61% 11959Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 15655m 52% 2963Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20821m 69% 10902Mi 9% gke-xlou-cdm-ds-0f601751-j488 8722m 29% 7225Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18596m 62% 15879Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 8590m 28% 7845Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20076m 67% 13757Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 9738m 61% 6395Mi 10% 20:26:00 DEBUG --- stderr --- 20:26:00 DEBUG 20:26:41 WARNING File /tmp/lodemon_data-20240420_191838/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [20/Apr/2024 20:26:41] "GET /monitoring/average?start_time=20240420_185739&stop_time=20240420_192441 HTTP/1.1" 200 - 20:26:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713641215" 20:26:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641215" 20:26:55 INFO [http_cmd]: http status code OK 20:26:55 DEBUG --- status code --- 20:26:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:55 DEBUG --- http response --- 20:26:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:26:55 ERROR [http_cmd]: ERROR 20:26:55 ERROR --- status code --- 20:26:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:26:55 ERROR --- http response --- 20:26:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:26:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641216" 20:26:56 INFO [http_cmd]: http status code OK 20:26:56 DEBUG --- status code --- 20:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:56 DEBUG --- http response --- 20:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:26:56 DEBUG ----- output ----- 20:26:56 DEBUG { 20:26:56 DEBUG "status": "success", 20:26:56 DEBUG "data": { 20:26:56 DEBUG "resultType": "vector", 20:26:56 DEBUG "result": [ 20:26:56 DEBUG { 20:26:56 DEBUG "metric": { 20:26:56 DEBUG "pod": "ds-idrepo-0" 20:26:56 DEBUG }, 20:26:56 DEBUG "value": [ 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG [...] 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG "pod": "ds-idrepo-2" 20:26:56 DEBUG }, 20:26:56 DEBUG "value": [ 20:26:56 DEBUG 1713641216, 20:26:56 DEBUG "0" 20:26:56 DEBUG ] 20:26:56 DEBUG } 20:26:56 DEBUG ] 20:26:56 DEBUG } 20:26:56 DEBUG } 20:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641216" 20:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641216" 20:26:56 INFO [http_cmd]: http status code OK 20:26:56 INFO [http_cmd]: http status code OK 20:26:56 DEBUG --- status code --- 20:26:56 DEBUG --- status code --- 20:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:56 DEBUG --- http response --- 20:26:56 DEBUG --- http response --- 20:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:26:56 DEBUG ----- output ----- 20:26:56 DEBUG ----- output ----- 20:26:56 DEBUG { 20:26:56 DEBUG { 20:26:56 DEBUG "status": "success", 20:26:56 DEBUG "status": "success", 20:26:56 DEBUG "data": { 20:26:56 DEBUG "data": { 20:26:56 DEBUG "resultType": "vector", 20:26:56 DEBUG "resultType": "vector", 20:26:56 DEBUG "result": [ 20:26:56 DEBUG "result": [ 20:26:56 DEBUG { 20:26:56 DEBUG { 20:26:56 DEBUG "metric": { 20:26:56 DEBUG "metric": { 20:26:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:26:56 DEBUG "pod": "ds-idrepo-0" 20:26:56 DEBUG }, 20:26:56 DEBUG }, 20:26:56 DEBUG "value": [ 20:26:56 DEBUG "value": [ 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG [...] 20:26:56 DEBUG [...] 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:26:56 DEBUG "pod": "ds-idrepo-2" 20:26:56 DEBUG }, 20:26:56 DEBUG }, 20:26:56 DEBUG "value": [ 20:26:56 DEBUG "value": [ 20:26:56 DEBUG 1713641216, 20:26:56 DEBUG 1713641216, 20:26:56 DEBUG "2835.2666666666664" 20:26:56 DEBUG "0" 20:26:56 DEBUG ] 20:26:56 DEBUG ] 20:26:56 DEBUG } 20:26:56 DEBUG } 20:26:56 DEBUG ] 20:26:56 DEBUG ] 20:26:56 DEBUG } 20:26:56 DEBUG } 20:26:56 DEBUG } 20:26:56 DEBUG } 20:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641216" 20:26:56 INFO [http_cmd]: http status code OK 20:26:56 DEBUG --- status code --- 20:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:56 DEBUG --- http response --- 20:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:26:56 DEBUG ----- output ----- 20:26:56 DEBUG { 20:26:56 DEBUG "status": "success", 20:26:56 DEBUG "data": { 20:26:56 DEBUG "resultType": "vector", 20:26:56 DEBUG "result": [ 20:26:56 DEBUG { 20:26:56 DEBUG "metric": { 20:26:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:26:56 DEBUG }, 20:26:56 DEBUG "value": [ 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG [...] 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG 20:26:56 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:26:56 DEBUG }, 20:26:56 DEBUG "value": [ 20:26:56 DEBUG 1713641216, 20:26:56 DEBUG "1168938.3320141037" 20:26:56 DEBUG ] 20:26:56 DEBUG } 20:26:56 DEBUG ] 20:26:56 DEBUG } 20:26:56 DEBUG } 20:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641216" 20:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713641216" 20:26:57 INFO [http_cmd]: http status code OK 20:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641217" 20:26:57 DEBUG --- status code --- 20:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:57 DEBUG --- http response --- 20:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:26:57 DEBUG ----- output ----- 20:26:57 DEBUG { 20:26:57 DEBUG "status": "success", 20:26:57 DEBUG "data": { 20:26:57 DEBUG "resultType": "vector", 20:26:57 DEBUG "result": [ 20:26:57 DEBUG { 20:26:57 DEBUG "metric": { 20:26:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:26:57 DEBUG }, 20:26:57 DEBUG "value": [ 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG [...] 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:26:57 DEBUG }, 20:26:57 DEBUG "value": [ 20:26:57 DEBUG 1713641216, 20:26:57 DEBUG "0" 20:26:57 DEBUG ] 20:26:57 DEBUG } 20:26:57 DEBUG ] 20:26:57 DEBUG } 20:26:57 DEBUG } 20:26:57 INFO [http_cmd]: http status code OK 20:26:57 DEBUG --- status code --- 20:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:57 DEBUG --- http response --- 20:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:26:57 DEBUG ----- output ----- 20:26:57 DEBUG { 20:26:57 DEBUG "status": "success", 20:26:57 DEBUG "data": { 20:26:57 DEBUG "resultType": "vector", 20:26:57 DEBUG "result": [ 20:26:57 DEBUG { 20:26:57 DEBUG "metric": { 20:26:57 DEBUG "instance": "10.142.0.107:9100" 20:26:57 DEBUG }, 20:26:57 DEBUG "value": [ 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG [...] 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG "instance": "10.142.0.43:9100" 20:26:57 DEBUG }, 20:26:57 DEBUG "value": [ 20:26:57 DEBUG 1713641217, 20:26:57 DEBUG "3759490503680" 20:26:57 DEBUG ] 20:26:57 DEBUG } 20:26:57 DEBUG ] 20:26:57 DEBUG } 20:26:57 DEBUG } 20:26:57 INFO [http_cmd]: http status code OK 20:26:57 DEBUG --- status code --- 20:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:57 DEBUG --- http response --- 20:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:26:57 DEBUG ----- output ----- 20:26:57 DEBUG { 20:26:57 DEBUG "status": "success", 20:26:57 DEBUG "data": { 20:26:57 DEBUG "resultType": "vector", 20:26:57 DEBUG "result": [ 20:26:57 DEBUG { 20:26:57 DEBUG "metric": { 20:26:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:26:57 DEBUG }, 20:26:57 DEBUG "value": [ 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG [...] 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:26:57 DEBUG }, 20:26:57 DEBUG "value": [ 20:26:57 DEBUG 1713641216, 20:26:57 DEBUG "6.928307220194929" 20:26:57 DEBUG ] 20:26:57 DEBUG } 20:26:57 DEBUG ] 20:26:57 DEBUG } 20:26:57 DEBUG } 20:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641217" 20:26:57 INFO [http_cmd]: http status code OK 20:26:57 DEBUG --- status code --- 20:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:57 DEBUG --- http response --- 20:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:26:57 DEBUG ----- output ----- 20:26:57 DEBUG { 20:26:57 DEBUG "status": "success", 20:26:57 DEBUG "data": { 20:26:57 DEBUG "resultType": "vector", 20:26:57 DEBUG "result": [ 20:26:57 DEBUG { 20:26:57 DEBUG "metric": { 20:26:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:26:57 DEBUG }, 20:26:57 DEBUG "value": [ 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG [...] 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:26:57 DEBUG }, 20:26:57 DEBUG "value": [ 20:26:57 DEBUG 1713641217, 20:26:57 DEBUG "0" 20:26:57 DEBUG ] 20:26:57 DEBUG } 20:26:57 DEBUG ] 20:26:57 DEBUG } 20:26:57 DEBUG } 20:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641217" 20:26:57 INFO [http_cmd]: http status code OK 20:26:57 DEBUG --- status code --- 20:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:57 DEBUG --- http response --- 20:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:26:57 DEBUG ----- output ----- 20:26:57 DEBUG { 20:26:57 DEBUG "status": "success", 20:26:57 DEBUG "data": { 20:26:57 DEBUG "resultType": "vector", 20:26:57 DEBUG "result": [ 20:26:57 DEBUG { 20:26:57 DEBUG "metric": { 20:26:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:26:57 DEBUG }, 20:26:57 DEBUG "value": [ 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG [...] 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG 20:26:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:26:57 DEBUG }, 20:26:57 DEBUG "value": [ 20:26:57 DEBUG 1713641217, 20:26:57 DEBUG "0" 20:26:57 DEBUG ] 20:26:57 DEBUG } 20:26:57 DEBUG ] 20:26:57 DEBUG } 20:26:57 DEBUG } 20:26:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641218" 20:26:58 INFO [http_cmd]: http status code OK 20:26:58 DEBUG --- status code --- 20:26:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:58 DEBUG --- http response --- 20:26:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:26:58 DEBUG ----- output ----- 20:26:58 DEBUG { 20:26:58 DEBUG "status": "success", 20:26:58 DEBUG "data": { 20:26:58 DEBUG "resultType": "vector", 20:26:58 DEBUG "result": [ 20:26:58 DEBUG { 20:26:58 DEBUG "metric": { 20:26:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:26:58 DEBUG }, 20:26:58 DEBUG "value": [ 20:26:58 DEBUG 20:26:58 DEBUG 20:26:58 DEBUG 20:26:58 DEBUG [...] 20:26:58 DEBUG 20:26:58 DEBUG 20:26:58 DEBUG 20:26:58 DEBUG "pod": "lodemon-6f4955bbbb-clfwf" 20:26:58 DEBUG }, 20:26:58 DEBUG "value": [ 20:26:58 DEBUG 1713641218, 20:26:58 DEBUG "0.0023032538677532056" 20:26:58 DEBUG ] 20:26:58 DEBUG } 20:26:58 DEBUG ] 20:26:58 DEBUG } 20:26:58 DEBUG } 20:26:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641218" 20:26:58 INFO [http_cmd]: http status code OK 20:26:58 DEBUG --- status code --- 20:26:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:58 DEBUG --- http response --- 20:26:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:26:58 DEBUG ----- output ----- 20:26:58 DEBUG { 20:26:58 DEBUG "status": "success", 20:26:58 DEBUG "data": { 20:26:58 DEBUG "resultType": "vector", 20:26:58 DEBUG "result": [ 20:26:58 DEBUG { 20:26:58 DEBUG "metric": { 20:26:58 DEBUG "pod": "ds-idrepo-0" 20:26:58 DEBUG }, 20:26:58 DEBUG "value": [ 20:26:58 DEBUG 20:26:58 DEBUG 20:26:58 DEBUG 20:26:58 DEBUG [...] 20:26:58 DEBUG 20:26:58 DEBUG 20:26:58 DEBUG 20:26:58 DEBUG "pod": "ds-idrepo-2" 20:26:58 DEBUG }, 20:26:58 DEBUG "value": [ 20:26:58 DEBUG 1713641218, 20:26:58 DEBUG "0" 20:26:58 DEBUG ] 20:26:58 DEBUG } 20:26:58 DEBUG ] 20:26:58 DEBUG } 20:26:58 DEBUG } 20:26:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641218" 20:26:58 INFO [http_cmd]: http status code OK 20:26:58 DEBUG --- status code --- 20:26:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:58 DEBUG --- http response --- 20:26:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:26:58 DEBUG ----- output ----- 20:26:58 DEBUG { 20:26:58 DEBUG "status": "success", 20:26:58 DEBUG "data": { 20:26:58 DEBUG "resultType": "vector", 20:26:58 DEBUG "result": [ 20:26:58 DEBUG { 20:26:58 DEBUG "metric": { 20:26:58 DEBUG "pod": "ds-cts-0" 20:26:58 DEBUG }, 20:26:58 DEBUG "value": [ 20:26:58 DEBUG 20:26:58 DEBUG 20:26:58 DEBUG 20:26:58 DEBUG [...] 20:26:58 DEBUG 20:26:58 DEBUG 20:26:58 DEBUG 20:26:58 DEBUG "pod": "ds-cts-2" 20:26:58 DEBUG }, 20:26:58 DEBUG "value": [ 20:26:58 DEBUG 1713641218, 20:26:58 DEBUG "0" 20:26:58 DEBUG ] 20:26:58 DEBUG } 20:26:58 DEBUG ] 20:26:58 DEBUG } 20:26:58 DEBUG } 20:26:58 INFO 20:26:58 INFO [loop_until]: kubectl --namespace=xlou top pods 20:26:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:26:58 INFO [loop_until]: OK (rc = 0) 20:26:58 DEBUG --- stdout --- 20:26:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 3587m 8661Mi am-767f6b67b5-qcqz2 4869m 9070Mi am-767f6b67b5-vbltz 3211m 8669Mi ds-cts-0 8269m 11030Mi ds-cts-1 8100m 10820Mi ds-cts-2 8272m 11096Mi ds-idrepo-0 788m 3894Mi ds-idrepo-1 1677m 3886Mi ds-idrepo-2 2579m 3828Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 3m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 1351m 250Mi 20:26:58 DEBUG --- stderr --- 20:26:58 DEBUG 20:26:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713641218" 20:26:59 INFO [http_cmd]: http status code OK 20:26:59 DEBUG --- status code --- 20:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:59 DEBUG --- http response --- 20:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:26:59 DEBUG ----- output ----- 20:26:59 DEBUG { 20:26:59 DEBUG "status": "success", 20:26:59 DEBUG "data": { 20:26:59 DEBUG "resultType": "vector", 20:26:59 DEBUG "result": [ 20:26:59 DEBUG { 20:26:59 DEBUG "metric": { 20:26:59 DEBUG "pod": "ds-idrepo-0" 20:26:59 DEBUG }, 20:26:59 DEBUG "value": [ 20:26:59 DEBUG 20:26:59 DEBUG 20:26:59 DEBUG 20:26:59 DEBUG [...] 20:26:59 DEBUG 20:26:59 DEBUG 20:26:59 DEBUG 20:26:59 DEBUG "pod": "ds-cts-2" 20:26:59 DEBUG }, 20:26:59 DEBUG "value": [ 20:26:59 DEBUG 1713641218, 20:26:59 DEBUG "0" 20:26:59 DEBUG ] 20:26:59 DEBUG } 20:26:59 DEBUG ] 20:26:59 DEBUG } 20:26:59 DEBUG } 20:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641219" 20:26:59 INFO [http_cmd]: http status code OK 20:26:59 DEBUG --- status code --- 20:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:59 DEBUG --- http response --- 20:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:26:59 DEBUG ----- output ----- 20:26:59 DEBUG { 20:26:59 DEBUG "status": "success", 20:26:59 DEBUG "data": { 20:26:59 DEBUG "resultType": "vector", 20:26:59 DEBUG "result": [ 20:26:59 DEBUG { 20:26:59 DEBUG "metric": { 20:26:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:26:59 DEBUG }, 20:26:59 DEBUG "value": [ 20:26:59 DEBUG 20:26:59 DEBUG 20:26:59 DEBUG 20:26:59 DEBUG [...] 20:26:59 DEBUG 20:26:59 DEBUG 20:26:59 DEBUG 20:26:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:26:59 DEBUG }, 20:26:59 DEBUG "value": [ 20:26:59 DEBUG 1713641219, 20:26:59 DEBUG "462377.5" 20:26:59 DEBUG ] 20:26:59 DEBUG } 20:26:59 DEBUG ] 20:26:59 DEBUG } 20:26:59 DEBUG } 20:26:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641219" 20:26:59 INFO [http_cmd]: http status code OK 20:26:59 DEBUG --- status code --- 20:26:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:26:59 DEBUG --- http response --- 20:26:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:26:59 DEBUG ----- output ----- 20:26:59 DEBUG { 20:26:59 DEBUG "status": "success", 20:26:59 DEBUG "data": { 20:26:59 DEBUG "resultType": "vector", 20:26:59 DEBUG "result": [ 20:26:59 DEBUG { 20:26:59 DEBUG "metric": { 20:26:59 DEBUG "instance": "10.142.0.107:9100" 20:26:59 DEBUG }, 20:26:59 DEBUG "value": [ 20:26:59 DEBUG 20:26:59 DEBUG 20:26:59 DEBUG 20:26:59 DEBUG [...] 20:26:59 DEBUG 20:26:59 DEBUG 20:26:59 DEBUG 20:26:59 DEBUG "instance": "10.142.0.43:9100" 20:26:59 DEBUG }, 20:26:59 DEBUG "value": [ 20:26:59 DEBUG 1713641219, 20:26:59 DEBUG "0.25349999999999207" 20:26:59 DEBUG ] 20:26:59 DEBUG } 20:26:59 DEBUG ] 20:26:59 DEBUG } 20:26:59 DEBUG } 20:27:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641220" 20:27:00 INFO [http_cmd]: http status code OK 20:27:00 DEBUG --- status code --- 20:27:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:00 DEBUG --- http response --- 20:27:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:27:00 DEBUG ----- output ----- 20:27:00 DEBUG { 20:27:00 DEBUG "status": "success", 20:27:00 DEBUG "data": { 20:27:00 DEBUG "resultType": "vector", 20:27:00 DEBUG "result": [ 20:27:00 DEBUG { 20:27:00 DEBUG "metric": { 20:27:00 DEBUG "instance": "10.142.0.107:9100" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG [...] 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG "instance": "10.142.0.43:9100" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 1713641220, 20:27:00 DEBUG "42926073.266666666" 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG } 20:27:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713641220" 20:27:00 INFO [http_cmd]: http status code OK 20:27:00 DEBUG --- status code --- 20:27:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:00 DEBUG --- http response --- 20:27:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:27:00 DEBUG ----- output ----- 20:27:00 DEBUG { 20:27:00 DEBUG "status": "success", 20:27:00 DEBUG "data": { 20:27:00 DEBUG "resultType": "vector", 20:27:00 DEBUG "result": [ 20:27:00 DEBUG { 20:27:00 DEBUG "metric": { 20:27:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG [...] 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 1713641220, 20:27:00 DEBUG "358916096" 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG } 20:27:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641220" 20:27:00 INFO [http_cmd]: http status code OK 20:27:00 DEBUG --- status code --- 20:27:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:00 DEBUG --- http response --- 20:27:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:27:00 DEBUG ----- output ----- 20:27:00 DEBUG { 20:27:00 DEBUG "status": "success", 20:27:00 DEBUG "data": { 20:27:00 DEBUG "resultType": "vector", 20:27:00 DEBUG "result": [ 20:27:00 DEBUG { 20:27:00 DEBUG "metric": { 20:27:00 DEBUG "instance": "10.142.0.107:9100" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG [...] 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG "instance": "10.142.0.43:9100" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 1713641220, 20:27:00 DEBUG "42930831" 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG } 20:27:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641220" 20:27:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641220" 20:27:00 INFO [http_cmd]: http status code OK 20:27:00 DEBUG --- status code --- 20:27:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:00 DEBUG --- http response --- 20:27:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:27:00 DEBUG ----- output ----- 20:27:00 INFO [http_cmd]: http status code OK 20:27:00 DEBUG { 20:27:00 DEBUG --- status code --- 20:27:00 DEBUG "status": "success", 20:27:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:00 DEBUG "data": { 20:27:00 DEBUG --- http response --- 20:27:00 DEBUG "resultType": "vector", 20:27:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:27:00 DEBUG "result": [ 20:27:00 DEBUG ----- output ----- 20:27:00 DEBUG { 20:27:00 DEBUG { 20:27:00 DEBUG "metric": { 20:27:00 DEBUG "status": "success", 20:27:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:27:00 DEBUG "data": { 20:27:00 DEBUG }, 20:27:00 DEBUG "resultType": "vector", 20:27:00 DEBUG "value": [ 20:27:00 DEBUG "result": [ 20:27:00 DEBUG 20:27:00 DEBUG { 20:27:00 DEBUG 20:27:00 DEBUG "metric": { 20:27:00 DEBUG 20:27:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:27:00 DEBUG [...] 20:27:00 DEBUG }, 20:27:00 DEBUG 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:27:00 DEBUG 20:27:00 DEBUG }, 20:27:00 DEBUG [...] 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 20:27:00 DEBUG 1713641220, 20:27:00 DEBUG 20:27:00 DEBUG "0" 20:27:00 DEBUG 20:27:00 DEBUG ] 20:27:00 DEBUG "pod": "lodemon-6f4955bbbb-clfwf" 20:27:00 DEBUG } 20:27:00 DEBUG }, 20:27:00 DEBUG ] 20:27:00 DEBUG "value": [ 20:27:00 DEBUG } 20:27:00 DEBUG 1713641220, 20:27:00 DEBUG } 20:27:00 DEBUG "1863868.3036943234" 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG } 20:27:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641220" 20:27:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641220" 20:27:00 INFO [http_cmd]: http status code OK 20:27:00 DEBUG --- status code --- 20:27:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:00 DEBUG --- http response --- 20:27:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:27:00 DEBUG ----- output ----- 20:27:00 DEBUG { 20:27:00 DEBUG "status": "success", 20:27:00 DEBUG "data": { 20:27:00 DEBUG "resultType": "vector", 20:27:00 DEBUG "result": [ 20:27:00 DEBUG { 20:27:00 INFO [http_cmd]: http status code OK 20:27:00 DEBUG "metric": { 20:27:00 DEBUG --- status code --- 20:27:00 DEBUG "pod": "ds-cts-0" 20:27:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:00 DEBUG }, 20:27:00 DEBUG --- http response --- 20:27:00 DEBUG "value": [ 20:27:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 20:27:00 DEBUG 20:27:00 DEBUG ----- output ----- 20:27:00 DEBUG 20:27:00 DEBUG { 20:27:00 DEBUG 20:27:00 DEBUG "status": "success", 20:27:00 DEBUG [...] 20:27:00 DEBUG "data": { 20:27:00 DEBUG 20:27:00 DEBUG "resultType": "vector", 20:27:00 DEBUG 20:27:00 DEBUG "result": [ 20:27:00 DEBUG 20:27:00 DEBUG { 20:27:00 DEBUG "pod": "ds-cts-2" 20:27:00 DEBUG "metric": { 20:27:00 DEBUG }, 20:27:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:27:00 DEBUG "value": [ 20:27:00 DEBUG }, 20:27:00 DEBUG 1713641220, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG "30" 20:27:00 DEBUG 20:27:00 DEBUG ] 20:27:00 DEBUG 20:27:00 DEBUG } 20:27:00 DEBUG 20:27:00 DEBUG ] 20:27:00 DEBUG [...] 20:27:00 DEBUG } 20:27:00 DEBUG 20:27:00 DEBUG } 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG "pod": "lodemon-6f4955bbbb-clfwf" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 1713641220, 20:27:00 DEBUG "0" 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG } 20:27:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713641220" 20:27:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641220" 20:27:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641220" 20:27:00 INFO [http_cmd]: http status code OK 20:27:00 DEBUG --- status code --- 20:27:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:00 DEBUG --- http response --- 20:27:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:27:00 DEBUG ----- output ----- 20:27:00 DEBUG { 20:27:00 DEBUG "status": "success", 20:27:00 DEBUG "data": { 20:27:00 DEBUG "resultType": "vector", 20:27:00 DEBUG "result": [ 20:27:00 DEBUG { 20:27:00 DEBUG "metric": { 20:27:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG [...] 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 1713641220, 20:27:00 DEBUG "0.159" 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG } 20:27:00 INFO [http_cmd]: http status code OK 20:27:00 DEBUG --- status code --- 20:27:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:00 DEBUG --- http response --- 20:27:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:27:00 DEBUG ----- output ----- 20:27:00 DEBUG { 20:27:00 DEBUG "status": "success", 20:27:00 DEBUG "data": { 20:27:00 DEBUG "resultType": "vector", 20:27:00 DEBUG "result": [ 20:27:00 DEBUG { 20:27:00 DEBUG "metric": { 20:27:00 DEBUG "instance": "10.142.0.107:9100" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG [...] 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG "instance": "10.142.0.43:9100" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 1713641220, 20:27:00 DEBUG "0.11866666666666295" 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG } 20:27:00 INFO [http_cmd]: http status code OK 20:27:00 DEBUG --- status code --- 20:27:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:00 DEBUG --- http response --- 20:27:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:27:00 DEBUG ----- output ----- 20:27:00 DEBUG { 20:27:00 DEBUG "status": "success", 20:27:00 DEBUG "data": { 20:27:00 DEBUG "resultType": "vector", 20:27:00 DEBUG "result": [ 20:27:00 DEBUG { 20:27:00 DEBUG "metric": { 20:27:00 DEBUG "pod": "ds-cts-0" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG [...] 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG 20:27:00 DEBUG "pod": "ds-cts-2" 20:27:00 DEBUG }, 20:27:00 DEBUG "value": [ 20:27:00 DEBUG 1713641220, 20:27:00 DEBUG "0.000033333333333333335" 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG ] 20:27:00 DEBUG } 20:27:00 DEBUG } 20:27:00 INFO 20:27:00 INFO [loop_until]: kubectl --namespace=xlou top node 20:27:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:27:00 INFO [loop_until]: OK (rc = 0) 20:27:00 DEBUG --- stdout --- 20:27:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6208Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 869m 2% 7636Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2845m 9% 11418Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 75m 0% 3343Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 8451m 28% 14236Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5175Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2903m 9% 11993Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 78m 0% 2939Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 3331m 11% 10934Mi 9% gke-xlou-cdm-ds-0f601751-j488 896m 3% 7228Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 7794m 26% 15875Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 1389m 4% 7853Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 9876m 33% 13763Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 101m 0% 5805Mi 9% 20:27:00 DEBUG --- stderr --- 20:27:00 DEBUG 20:27:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641275" 20:27:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713641275" 20:27:55 ERROR [http_cmd]: ERROR 20:27:55 INFO [http_cmd]: http status code OK 20:27:55 ERROR --- status code --- 20:27:55 DEBUG --- status code --- 20:27:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:27:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:55 ERROR --- http response --- 20:27:55 DEBUG --- http response --- 20:27:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:27:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:27:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641276" 20:27:56 INFO [http_cmd]: http status code OK 20:27:56 DEBUG --- status code --- 20:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:56 DEBUG --- http response --- 20:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:27:56 DEBUG ----- output ----- 20:27:56 DEBUG { 20:27:56 DEBUG "status": "success", 20:27:56 DEBUG "data": { 20:27:56 DEBUG "resultType": "vector", 20:27:56 DEBUG "result": [ 20:27:56 DEBUG { 20:27:56 DEBUG "metric": { 20:27:56 DEBUG "pod": "ds-idrepo-0" 20:27:56 DEBUG }, 20:27:56 DEBUG "value": [ 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG [...] 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG "pod": "ds-idrepo-2" 20:27:56 DEBUG }, 20:27:56 DEBUG "value": [ 20:27:56 DEBUG 1713641276, 20:27:56 DEBUG "0.3" 20:27:56 DEBUG ] 20:27:56 DEBUG } 20:27:56 DEBUG ] 20:27:56 DEBUG } 20:27:56 DEBUG } 20:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641276" 20:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641276" 20:27:56 INFO [http_cmd]: http status code OK 20:27:56 DEBUG --- status code --- 20:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:56 DEBUG --- http response --- 20:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:27:56 DEBUG ----- output ----- 20:27:56 DEBUG { 20:27:56 DEBUG "status": "success", 20:27:56 DEBUG "data": { 20:27:56 DEBUG "resultType": "vector", 20:27:56 DEBUG "result": [ 20:27:56 DEBUG { 20:27:56 DEBUG "metric": { 20:27:56 DEBUG "pod": "ds-idrepo-0" 20:27:56 DEBUG }, 20:27:56 DEBUG "value": [ 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG [...] 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG "pod": "ds-idrepo-2" 20:27:56 DEBUG }, 20:27:56 DEBUG "value": [ 20:27:56 DEBUG 1713641276, 20:27:56 DEBUG "0.3" 20:27:56 DEBUG ] 20:27:56 DEBUG } 20:27:56 DEBUG ] 20:27:56 DEBUG } 20:27:56 DEBUG } 20:27:56 INFO [http_cmd]: http status code OK 20:27:56 DEBUG --- status code --- 20:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:56 DEBUG --- http response --- 20:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:27:56 DEBUG ----- output ----- 20:27:56 DEBUG { 20:27:56 DEBUG "status": "success", 20:27:56 DEBUG "data": { 20:27:56 DEBUG "resultType": "vector", 20:27:56 DEBUG "result": [ 20:27:56 DEBUG { 20:27:56 DEBUG "metric": { 20:27:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:27:56 DEBUG }, 20:27:56 DEBUG "value": [ 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG [...] 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:27:56 DEBUG }, 20:27:56 DEBUG "value": [ 20:27:56 DEBUG 1713641276, 20:27:56 DEBUG "2824.4666666666667" 20:27:56 DEBUG ] 20:27:56 DEBUG } 20:27:56 DEBUG ] 20:27:56 DEBUG } 20:27:56 DEBUG } 20:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641276" 20:27:56 INFO [http_cmd]: http status code OK 20:27:56 DEBUG --- status code --- 20:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:56 DEBUG --- http response --- 20:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:27:56 DEBUG ----- output ----- 20:27:56 DEBUG { 20:27:56 DEBUG "status": "success", 20:27:56 DEBUG "data": { 20:27:56 DEBUG "resultType": "vector", 20:27:56 DEBUG "result": [ 20:27:56 DEBUG { 20:27:56 DEBUG "metric": { 20:27:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:27:56 DEBUG }, 20:27:56 DEBUG "value": [ 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG [...] 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG 20:27:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:27:56 DEBUG }, 20:27:56 DEBUG "value": [ 20:27:56 DEBUG 1713641276, 20:27:56 DEBUG "0" 20:27:56 DEBUG ] 20:27:56 DEBUG } 20:27:56 DEBUG ] 20:27:56 DEBUG } 20:27:56 DEBUG } 20:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641277" 20:27:57 INFO [http_cmd]: http status code OK 20:27:57 DEBUG --- status code --- 20:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:57 DEBUG --- http response --- 20:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:27:57 DEBUG ----- output ----- 20:27:57 DEBUG { 20:27:57 DEBUG "status": "success", 20:27:57 DEBUG "data": { 20:27:57 DEBUG "resultType": "vector", 20:27:57 DEBUG "result": [ 20:27:57 DEBUG { 20:27:57 DEBUG "metric": { 20:27:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:27:57 DEBUG }, 20:27:57 DEBUG "value": [ 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG [...] 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:27:57 DEBUG }, 20:27:57 DEBUG "value": [ 20:27:57 DEBUG 1713641277, 20:27:57 DEBUG "0" 20:27:57 DEBUG ] 20:27:57 DEBUG } 20:27:57 DEBUG ] 20:27:57 DEBUG } 20:27:57 DEBUG } 20:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641277" 20:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713641277" 20:27:57 INFO [http_cmd]: http status code OK 20:27:57 DEBUG --- status code --- 20:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:57 DEBUG --- http response --- 20:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:27:57 DEBUG ----- output ----- 20:27:57 DEBUG { 20:27:57 DEBUG "status": "success", 20:27:57 DEBUG "data": { 20:27:57 DEBUG "resultType": "vector", 20:27:57 DEBUG "result": [ 20:27:57 DEBUG { 20:27:57 DEBUG "metric": { 20:27:57 DEBUG "instance": "10.142.0.107:9100" 20:27:57 DEBUG }, 20:27:57 DEBUG "value": [ 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG [...] 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG "instance": "10.142.0.43:9100" 20:27:57 DEBUG }, 20:27:57 DEBUG "value": [ 20:27:57 DEBUG 1713641277, 20:27:57 DEBUG "3761948855296" 20:27:57 DEBUG ] 20:27:57 DEBUG } 20:27:57 DEBUG ] 20:27:57 DEBUG } 20:27:57 DEBUG } 20:27:57 INFO [http_cmd]: http status code OK 20:27:57 DEBUG --- status code --- 20:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:57 DEBUG --- http response --- 20:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:27:57 DEBUG ----- output ----- 20:27:57 DEBUG { 20:27:57 DEBUG "status": "success", 20:27:57 DEBUG "data": { 20:27:57 DEBUG "resultType": "vector", 20:27:57 DEBUG "result": [ 20:27:57 DEBUG { 20:27:57 DEBUG "metric": { 20:27:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:27:57 DEBUG }, 20:27:57 DEBUG "value": [ 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG [...] 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:27:57 DEBUG }, 20:27:57 DEBUG "value": [ 20:27:57 DEBUG 1713641277, 20:27:57 DEBUG "0.026775595614126953" 20:27:57 DEBUG ] 20:27:57 DEBUG } 20:27:57 DEBUG ] 20:27:57 DEBUG } 20:27:57 DEBUG } 20:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641277" 20:27:57 INFO [http_cmd]: http status code OK 20:27:57 DEBUG --- status code --- 20:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:57 DEBUG --- http response --- 20:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:27:57 DEBUG ----- output ----- 20:27:57 DEBUG { 20:27:57 DEBUG "status": "success", 20:27:57 DEBUG "data": { 20:27:57 DEBUG "resultType": "vector", 20:27:57 DEBUG "result": [ 20:27:57 DEBUG { 20:27:57 DEBUG "metric": { 20:27:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:27:57 DEBUG }, 20:27:57 DEBUG "value": [ 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG [...] 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:27:57 DEBUG }, 20:27:57 DEBUG "value": [ 20:27:57 DEBUG 1713641277, 20:27:57 DEBUG "0" 20:27:57 DEBUG ] 20:27:57 DEBUG } 20:27:57 DEBUG ] 20:27:57 DEBUG } 20:27:57 DEBUG } 20:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641277" 20:27:57 INFO [http_cmd]: http status code OK 20:27:57 DEBUG --- status code --- 20:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:57 DEBUG --- http response --- 20:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:27:57 DEBUG ----- output ----- 20:27:57 DEBUG { 20:27:57 DEBUG "status": "success", 20:27:57 DEBUG "data": { 20:27:57 DEBUG "resultType": "vector", 20:27:57 DEBUG "result": [ 20:27:57 DEBUG { 20:27:57 DEBUG "metric": { 20:27:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:27:57 DEBUG }, 20:27:57 DEBUG "value": [ 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG [...] 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG 20:27:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:27:57 DEBUG }, 20:27:57 DEBUG "value": [ 20:27:57 DEBUG 1713641277, 20:27:57 DEBUG "0" 20:27:57 DEBUG ] 20:27:57 DEBUG } 20:27:57 DEBUG ] 20:27:57 DEBUG } 20:27:57 DEBUG } 20:27:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641278" 20:27:58 INFO [http_cmd]: http status code OK 20:27:58 DEBUG --- status code --- 20:27:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:58 DEBUG --- http response --- 20:27:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:27:58 DEBUG ----- output ----- 20:27:58 DEBUG { 20:27:58 DEBUG "status": "success", 20:27:58 DEBUG "data": { 20:27:58 DEBUG "resultType": "vector", 20:27:58 DEBUG "result": [ 20:27:58 DEBUG { 20:27:58 DEBUG "metric": { 20:27:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:27:58 DEBUG }, 20:27:58 DEBUG "value": [ 20:27:58 DEBUG 20:27:58 DEBUG 20:27:58 DEBUG 20:27:58 DEBUG [...] 20:27:58 DEBUG 20:27:58 DEBUG 20:27:58 DEBUG 20:27:58 DEBUG "pod": "ds-cts-0" 20:27:58 DEBUG }, 20:27:58 DEBUG "value": [ 20:27:58 DEBUG 1713641278, 20:27:58 DEBUG "0" 20:27:58 DEBUG ] 20:27:58 DEBUG } 20:27:58 DEBUG ] 20:27:58 DEBUG } 20:27:58 DEBUG } 20:27:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641278" 20:27:58 INFO [http_cmd]: http status code OK 20:27:58 DEBUG --- status code --- 20:27:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:58 DEBUG --- http response --- 20:27:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:27:58 DEBUG ----- output ----- 20:27:58 DEBUG { 20:27:58 DEBUG "status": "success", 20:27:58 DEBUG "data": { 20:27:58 DEBUG "resultType": "vector", 20:27:58 DEBUG "result": [ 20:27:58 DEBUG { 20:27:58 DEBUG "metric": { 20:27:58 DEBUG "pod": "ds-idrepo-0" 20:27:58 DEBUG }, 20:27:58 DEBUG "value": [ 20:27:58 DEBUG 20:27:58 DEBUG 20:27:58 DEBUG 20:27:58 DEBUG [...] 20:27:58 DEBUG 20:27:58 DEBUG 20:27:58 DEBUG 20:27:58 DEBUG "pod": "ds-idrepo-2" 20:27:58 DEBUG }, 20:27:58 DEBUG "value": [ 20:27:58 DEBUG 1713641278, 20:27:58 DEBUG "0.3" 20:27:58 DEBUG ] 20:27:58 DEBUG } 20:27:58 DEBUG ] 20:27:58 DEBUG } 20:27:58 DEBUG } 20:27:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641278" 20:27:58 INFO [http_cmd]: http status code OK 20:27:58 DEBUG --- status code --- 20:27:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:58 DEBUG --- http response --- 20:27:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:27:58 DEBUG ----- output ----- 20:27:58 DEBUG { 20:27:58 DEBUG "status": "success", 20:27:58 DEBUG "data": { 20:27:58 DEBUG "resultType": "vector", 20:27:58 DEBUG "result": [ 20:27:58 DEBUG { 20:27:58 DEBUG "metric": { 20:27:58 DEBUG "pod": "ds-cts-0" 20:27:58 DEBUG }, 20:27:58 DEBUG "value": [ 20:27:58 DEBUG 20:27:58 DEBUG 20:27:58 DEBUG 20:27:58 DEBUG [...] 20:27:58 DEBUG 20:27:58 DEBUG 20:27:58 DEBUG 20:27:58 DEBUG "pod": "ds-cts-2" 20:27:58 DEBUG }, 20:27:58 DEBUG "value": [ 20:27:58 DEBUG 1713641278, 20:27:58 DEBUG "0" 20:27:58 DEBUG ] 20:27:58 DEBUG } 20:27:58 DEBUG ] 20:27:58 DEBUG } 20:27:58 DEBUG } 20:27:58 INFO 20:27:58 INFO [loop_until]: kubectl --namespace=xlou top pods 20:27:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:27:58 INFO [loop_until]: OK (rc = 0) 20:27:58 DEBUG --- stdout --- 20:27:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 2776m 8623Mi am-767f6b67b5-qcqz2 2822m 9031Mi am-767f6b67b5-vbltz 3108m 8625Mi ds-cts-0 8466m 10995Mi ds-cts-1 7633m 10847Mi ds-cts-2 8898m 11080Mi ds-idrepo-0 1527m 3924Mi ds-idrepo-1 1488m 3888Mi ds-idrepo-2 1449m 3845Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 7m 1179Mi idm-b85f9d978-wnhxb 4m 1238Mi lodemon-6f4955bbbb-clfwf 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 1213m 631Mi 20:27:58 DEBUG --- stderr --- 20:27:58 DEBUG 20:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713641279" 20:27:59 INFO [http_cmd]: http status code OK 20:27:59 DEBUG --- status code --- 20:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:59 DEBUG --- http response --- 20:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:27:59 DEBUG ----- output ----- 20:27:59 DEBUG { 20:27:59 DEBUG "status": "success", 20:27:59 DEBUG "data": { 20:27:59 DEBUG "resultType": "vector", 20:27:59 DEBUG "result": [ 20:27:59 DEBUG { 20:27:59 DEBUG "metric": { 20:27:59 DEBUG "pod": "ds-idrepo-0" 20:27:59 DEBUG }, 20:27:59 DEBUG "value": [ 20:27:59 DEBUG 20:27:59 DEBUG 20:27:59 DEBUG 20:27:59 DEBUG [...] 20:27:59 DEBUG 20:27:59 DEBUG 20:27:59 DEBUG 20:27:59 DEBUG "pod": "ds-cts-2" 20:27:59 DEBUG }, 20:27:59 DEBUG "value": [ 20:27:59 DEBUG 1713641279, 20:27:59 DEBUG "0" 20:27:59 DEBUG ] 20:27:59 DEBUG } 20:27:59 DEBUG ] 20:27:59 DEBUG } 20:27:59 DEBUG } 20:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641279" 20:27:59 INFO [http_cmd]: http status code OK 20:27:59 DEBUG --- status code --- 20:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:59 DEBUG --- http response --- 20:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:27:59 DEBUG ----- output ----- 20:27:59 DEBUG { 20:27:59 DEBUG "status": "success", 20:27:59 DEBUG "data": { 20:27:59 DEBUG "resultType": "vector", 20:27:59 DEBUG "result": [ 20:27:59 DEBUG { 20:27:59 DEBUG "metric": { 20:27:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:27:59 DEBUG }, 20:27:59 DEBUG "value": [ 20:27:59 DEBUG 20:27:59 DEBUG 20:27:59 DEBUG 20:27:59 DEBUG [...] 20:27:59 DEBUG 20:27:59 DEBUG 20:27:59 DEBUG 20:27:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:27:59 DEBUG }, 20:27:59 DEBUG "value": [ 20:27:59 DEBUG 1713641279, 20:27:59 DEBUG "292530.5" 20:27:59 DEBUG ] 20:27:59 DEBUG } 20:27:59 DEBUG ] 20:27:59 DEBUG } 20:27:59 DEBUG } 20:27:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641279" 20:27:59 INFO [http_cmd]: http status code OK 20:27:59 DEBUG --- status code --- 20:27:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:27:59 DEBUG --- http response --- 20:27:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:27:59 DEBUG ----- output ----- 20:27:59 DEBUG { 20:27:59 DEBUG "status": "success", 20:27:59 DEBUG "data": { 20:27:59 DEBUG "resultType": "vector", 20:27:59 DEBUG "result": [ 20:27:59 DEBUG { 20:27:59 DEBUG "metric": { 20:27:59 DEBUG "instance": "10.142.0.107:9100" 20:27:59 DEBUG }, 20:27:59 DEBUG "value": [ 20:27:59 DEBUG 20:27:59 DEBUG 20:27:59 DEBUG 20:27:59 DEBUG [...] 20:27:59 DEBUG 20:27:59 DEBUG 20:27:59 DEBUG 20:27:59 DEBUG "instance": "10.142.0.43:9100" 20:27:59 DEBUG }, 20:27:59 DEBUG "value": [ 20:27:59 DEBUG 1713641279, 20:27:59 DEBUG "0.23743333333331879" 20:27:59 DEBUG ] 20:27:59 DEBUG } 20:27:59 DEBUG ] 20:27:59 DEBUG } 20:27:59 DEBUG } 20:28:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641280" 20:28:00 INFO [http_cmd]: http status code OK 20:28:00 DEBUG --- status code --- 20:28:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:00 DEBUG --- http response --- 20:28:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:28:00 DEBUG ----- output ----- 20:28:00 DEBUG { 20:28:00 DEBUG "status": "success", 20:28:00 DEBUG "data": { 20:28:00 DEBUG "resultType": "vector", 20:28:00 DEBUG "result": [ 20:28:00 DEBUG { 20:28:00 DEBUG "metric": { 20:28:00 DEBUG "instance": "10.142.0.107:9100" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG [...] 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG "instance": "10.142.0.43:9100" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 1713641280, 20:28:00 DEBUG "18527307.666666664" 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG } 20:28:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713641280" 20:28:00 INFO [http_cmd]: http status code OK 20:28:00 DEBUG --- status code --- 20:28:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:00 DEBUG --- http response --- 20:28:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:28:00 DEBUG ----- output ----- 20:28:00 DEBUG { 20:28:00 DEBUG "status": "success", 20:28:00 DEBUG "data": { 20:28:00 DEBUG "resultType": "vector", 20:28:00 DEBUG "result": [ 20:28:00 DEBUG { 20:28:00 DEBUG "metric": { 20:28:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG [...] 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 1713641280, 20:28:00 DEBUG "360677376" 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG } 20:28:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641280" 20:28:00 INFO [http_cmd]: http status code OK 20:28:00 DEBUG --- status code --- 20:28:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:00 DEBUG --- http response --- 20:28:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:28:00 DEBUG ----- output ----- 20:28:00 DEBUG { 20:28:00 DEBUG "status": "success", 20:28:00 DEBUG "data": { 20:28:00 DEBUG "resultType": "vector", 20:28:00 DEBUG "result": [ 20:28:00 DEBUG { 20:28:00 DEBUG "metric": { 20:28:00 DEBUG "instance": "10.142.0.107:9100" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG [...] 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG "instance": "10.142.0.43:9100" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 1713641280, 20:28:00 DEBUG "18533194.966666665" 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG } 20:28:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641280" 20:28:00 INFO [http_cmd]: http status code OK 20:28:00 DEBUG --- status code --- 20:28:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:00 DEBUG --- http response --- 20:28:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:28:00 DEBUG ----- output ----- 20:28:00 DEBUG { 20:28:00 DEBUG "status": "success", 20:28:00 DEBUG "data": { 20:28:00 DEBUG "resultType": "vector", 20:28:00 DEBUG "result": [ 20:28:00 DEBUG { 20:28:00 DEBUG "metric": { 20:28:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG [...] 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 1713641280, 20:28:00 DEBUG "0" 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG } 20:28:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641280" 20:28:00 INFO [http_cmd]: http status code OK 20:28:00 DEBUG --- status code --- 20:28:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:00 DEBUG --- http response --- 20:28:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:28:00 DEBUG ----- output ----- 20:28:00 DEBUG { 20:28:00 DEBUG "status": "success", 20:28:00 DEBUG "data": { 20:28:00 DEBUG "resultType": "vector", 20:28:00 DEBUG "result": [ 20:28:00 DEBUG { 20:28:00 DEBUG "metric": { 20:28:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG [...] 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 1713641280, 20:28:00 DEBUG "0" 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG } 20:28:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641280" 20:28:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713641280" 20:28:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641280" 20:28:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641280" 20:28:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641280" 20:28:00 INFO [http_cmd]: http status code OK 20:28:00 DEBUG --- status code --- 20:28:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:00 DEBUG --- http response --- 20:28:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:28:00 DEBUG ----- output ----- 20:28:00 DEBUG { 20:28:00 DEBUG "status": "success", 20:28:00 DEBUG "data": { 20:28:00 DEBUG "resultType": "vector", 20:28:00 DEBUG "result": [ 20:28:00 DEBUG { 20:28:00 DEBUG "metric": { 20:28:00 DEBUG "pod": "ds-cts-0" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG [...] 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG "pod": "ds-cts-2" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 1713641280, 20:28:00 DEBUG "30" 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG } 20:28:00 INFO [http_cmd]: http status code OK 20:28:00 DEBUG --- status code --- 20:28:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:00 DEBUG --- http response --- 20:28:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:28:00 DEBUG ----- output ----- 20:28:00 DEBUG { 20:28:00 DEBUG "status": "success", 20:28:00 DEBUG "data": { 20:28:00 DEBUG "resultType": "vector", 20:28:00 DEBUG "result": [ 20:28:00 DEBUG { 20:28:00 DEBUG "metric": { 20:28:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG [...] 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 1713641280, 20:28:00 DEBUG "0.159" 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG } 20:28:00 INFO [http_cmd]: http status code OK 20:28:00 INFO [http_cmd]: http status code OK 20:28:00 DEBUG --- status code --- 20:28:00 DEBUG --- status code --- 20:28:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:00 DEBUG --- http response --- 20:28:00 DEBUG --- http response --- 20:28:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:28:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:28:00 DEBUG ----- output ----- 20:28:00 DEBUG ----- output ----- 20:28:00 DEBUG { 20:28:00 DEBUG { 20:28:00 DEBUG "status": "success", 20:28:00 DEBUG "status": "success", 20:28:00 DEBUG "data": { 20:28:00 DEBUG "data": { 20:28:00 DEBUG "resultType": "vector", 20:28:00 DEBUG "resultType": "vector", 20:28:00 DEBUG "result": [ 20:28:00 DEBUG "result": [ 20:28:00 DEBUG { 20:28:00 DEBUG { 20:28:00 DEBUG "metric": { 20:28:00 DEBUG "metric": { 20:28:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:28:00 DEBUG "pod": "ds-cts-0" 20:28:00 DEBUG }, 20:28:00 DEBUG }, 20:28:00 INFO [http_cmd]: http status code OK 20:28:00 DEBUG "value": [ 20:28:00 DEBUG "value": [ 20:28:00 DEBUG --- status code --- 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG --- http response --- 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG [...] 20:28:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:28:00 DEBUG [...] 20:28:00 DEBUG 20:28:00 DEBUG ----- output ----- 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG { 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG "status": "success", 20:28:00 DEBUG 20:28:00 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:28:00 DEBUG "data": { 20:28:00 DEBUG "pod": "ds-cts-2" 20:28:00 DEBUG }, 20:28:00 DEBUG "resultType": "vector", 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG "result": [ 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 1713641280, 20:28:00 DEBUG { 20:28:00 DEBUG 1713641280, 20:28:00 DEBUG "0" 20:28:00 DEBUG "metric": { 20:28:00 DEBUG "0.000020437777777777778" 20:28:00 DEBUG ] 20:28:00 DEBUG "instance": "10.142.0.107:9100" 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG }, 20:28:00 DEBUG } 20:28:00 DEBUG ] 20:28:00 DEBUG "value": [ 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG 20:28:00 DEBUG } 20:28:00 DEBUG } 20:28:00 DEBUG 20:28:00 DEBUG } 20:28:00 DEBUG 20:28:00 DEBUG [...] 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG 20:28:00 DEBUG "instance": "10.142.0.43:9100" 20:28:00 DEBUG }, 20:28:00 DEBUG "value": [ 20:28:00 DEBUG 1713641280, 20:28:00 DEBUG "0.13966666666667799" 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG ] 20:28:00 DEBUG } 20:28:00 DEBUG } 20:28:00 INFO 20:28:00 INFO [loop_until]: kubectl --namespace=xlou top node 20:28:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:28:00 INFO [loop_until]: OK (rc = 0) 20:28:00 DEBUG --- stdout --- 20:28:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6213Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 1541m 5% 7618Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2724m 9% 11371Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3362Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 8318m 27% 14241Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5215Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2825m 9% 11953Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 82m 0% 2947Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 3281m 10% 10872Mi 9% gke-xlou-cdm-ds-0f601751-j488 1487m 4% 7212Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 7628m 25% 15890Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 1530m 5% 7834Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 8681m 29% 13759Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 1182m 7% 6299Mi 10% 20:28:00 DEBUG --- stderr --- 20:28:00 DEBUG 20:28:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713641335" 20:28:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641335" 20:28:55 ERROR [http_cmd]: ERROR 20:28:55 INFO [http_cmd]: http status code OK 20:28:55 ERROR --- status code --- 20:28:55 DEBUG --- status code --- 20:28:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:28:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:55 ERROR --- http response --- 20:28:55 DEBUG --- http response --- 20:28:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:28:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:28:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641336" 20:28:56 INFO [http_cmd]: http status code OK 20:28:56 DEBUG --- status code --- 20:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:56 DEBUG --- http response --- 20:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:28:56 DEBUG ----- output ----- 20:28:56 DEBUG { 20:28:56 DEBUG "status": "success", 20:28:56 DEBUG "data": { 20:28:56 DEBUG "resultType": "vector", 20:28:56 DEBUG "result": [ 20:28:56 DEBUG { 20:28:56 DEBUG "metric": { 20:28:56 DEBUG "pod": "ds-idrepo-0" 20:28:56 DEBUG }, 20:28:56 DEBUG "value": [ 20:28:56 DEBUG 20:28:56 DEBUG 20:28:56 DEBUG 20:28:56 DEBUG [...] 20:28:56 DEBUG 20:28:56 DEBUG 20:28:56 DEBUG 20:28:56 DEBUG "pod": "ds-idrepo-2" 20:28:56 DEBUG }, 20:28:56 DEBUG "value": [ 20:28:56 DEBUG 1713641336, 20:28:56 DEBUG "0" 20:28:56 DEBUG ] 20:28:56 DEBUG } 20:28:56 DEBUG ] 20:28:56 DEBUG } 20:28:56 DEBUG } 20:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641336" 20:28:56 INFO [http_cmd]: http status code OK 20:28:56 DEBUG --- status code --- 20:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:56 DEBUG --- http response --- 20:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:28:56 DEBUG ----- output ----- 20:28:56 DEBUG { 20:28:56 DEBUG "status": "success", 20:28:56 DEBUG "data": { 20:28:56 DEBUG "resultType": "vector", 20:28:56 DEBUG "result": [ 20:28:56 DEBUG { 20:28:56 DEBUG "metric": { 20:28:56 DEBUG "pod": "ds-idrepo-0" 20:28:56 DEBUG }, 20:28:56 DEBUG "value": [ 20:28:56 DEBUG 20:28:56 DEBUG 20:28:56 DEBUG 20:28:56 DEBUG [...] 20:28:56 DEBUG 20:28:56 DEBUG 20:28:56 DEBUG 20:28:56 DEBUG "pod": "ds-idrepo-2" 20:28:56 DEBUG }, 20:28:56 DEBUG "value": [ 20:28:56 DEBUG 1713641336, 20:28:56 DEBUG "0" 20:28:56 DEBUG ] 20:28:56 DEBUG } 20:28:56 DEBUG ] 20:28:56 DEBUG } 20:28:56 DEBUG } 20:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641336" 20:28:56 INFO [http_cmd]: http status code OK 20:28:56 DEBUG --- status code --- 20:28:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:56 DEBUG --- http response --- 20:28:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:28:56 DEBUG ----- output ----- 20:28:56 DEBUG { 20:28:56 DEBUG "status": "success", 20:28:56 DEBUG "data": { 20:28:56 DEBUG "resultType": "vector", 20:28:56 DEBUG "result": [ 20:28:56 DEBUG { 20:28:56 DEBUG "metric": { 20:28:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:28:56 DEBUG }, 20:28:56 DEBUG "value": [ 20:28:56 DEBUG 20:28:56 DEBUG 20:28:56 DEBUG 20:28:56 DEBUG [...] 20:28:56 DEBUG 20:28:56 DEBUG 20:28:56 DEBUG 20:28:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:28:56 DEBUG }, 20:28:56 DEBUG "value": [ 20:28:56 DEBUG 1713641336, 20:28:56 DEBUG "2826.9" 20:28:56 DEBUG ] 20:28:56 DEBUG } 20:28:56 DEBUG ] 20:28:56 DEBUG } 20:28:56 DEBUG } 20:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641336" 20:28:57 INFO [http_cmd]: http status code OK 20:28:57 DEBUG --- status code --- 20:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:57 DEBUG --- http response --- 20:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:28:57 DEBUG ----- output ----- 20:28:57 DEBUG { 20:28:57 DEBUG "status": "success", 20:28:57 DEBUG "data": { 20:28:57 DEBUG "resultType": "vector", 20:28:57 DEBUG "result": [ 20:28:57 DEBUG { 20:28:57 DEBUG "metric": { 20:28:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:28:57 DEBUG }, 20:28:57 DEBUG "value": [ 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG [...] 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:28:57 DEBUG }, 20:28:57 DEBUG "value": [ 20:28:57 DEBUG 1713641336, 20:28:57 DEBUG "0" 20:28:57 DEBUG ] 20:28:57 DEBUG } 20:28:57 DEBUG ] 20:28:57 DEBUG } 20:28:57 DEBUG } 20:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641337" 20:28:57 INFO [http_cmd]: http status code OK 20:28:57 DEBUG --- status code --- 20:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:57 DEBUG --- http response --- 20:28:57 [35mDEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:28:57 DEBUG ----- output ----- 20:28:57 DEBUG { 20:28:57 DEBUG "status": "success", 20:28:57 DEBUG "data": { 20:28:57 DEBUG "resultType": "vector", 20:28:57 DEBUG "result": [ 20:28:57 DEBUG { 20:28:57 DEBUG "metric": { 20:28:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:28:57 DEBUG }, 20:28:57 DEBUG "value": [ 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG [...] 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:28:57 DEBUG }, 20:28:57 DEBUG "value": [ 20:28:57 DEBUG 1713641337, 20:28:57 DEBUG "0" 20:28:57 DEBUG ] 20:28:57 DEBUG } 20:28:57 DEBUG ] 20:28:57 DEBUG } 20:28:57 DEBUG } 20:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641337" 20:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713641337" 20:28:57 INFO [http_cmd]: http status code OK 20:28:57 DEBUG --- status code --- 20:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:57 DEBUG --- http response --- 20:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:28:57 DEBUG ----- output ----- 20:28:57 DEBUG { 20:28:57 DEBUG "status": "success", 20:28:57 DEBUG "data": { 20:28:57 DEBUG "resultType": "vector", 20:28:57 DEBUG "result": [ 20:28:57 DEBUG { 20:28:57 DEBUG "metric": { 20:28:57 DEBUG "instance": "10.142.0.107:9100" 20:28:57 DEBUG }, 20:28:57 DEBUG "value": [ 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG [...] 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG "instance": "10.142.0.43:9100" 20:28:57 DEBUG }, 20:28:57 DEBUG "value": [ 20:28:57 DEBUG 1713641337, 20:28:57 DEBUG "3763274750976" 20:28:57 DEBUG ] 20:28:57 DEBUG } 20:28:57 DEBUG ] 20:28:57 DEBUG } 20:28:57 DEBUG } 20:28:57 INFO [http_cmd]: http status code OK 20:28:57 DEBUG --- status code --- 20:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:57 DEBUG --- http response --- 20:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:28:57 DEBUG ----- output ----- 20:28:57 DEBUG { 20:28:57 DEBUG "status": "success", 20:28:57 DEBUG "data": { 20:28:57 DEBUG "resultType": "vector", 20:28:57 DEBUG "result": [ 20:28:57 DEBUG { 20:28:57 DEBUG "metric": { 20:28:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:28:57 DEBUG }, 20:28:57 DEBUG "value": [ 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG [...] 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:28:57 DEBUG }, 20:28:57 DEBUG "value": [ 20:28:57 DEBUG 1713641337, 20:28:57 DEBUG "1.0275899636381485" 20:28:57 DEBUG ] 20:28:57 DEBUG } 20:28:57 DEBUG ] 20:28:57 DEBUG } 20:28:57 DEBUG } 20:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641337" 20:28:57 INFO [http_cmd]: http status code OK 20:28:57 DEBUG --- status code --- 20:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:57 DEBUG --- http response --- 20:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:28:57 DEBUG ----- output ----- 20:28:57 DEBUG { 20:28:57 DEBUG "status": "success", 20:28:57 DEBUG "data": { 20:28:57 DEBUG "resultType": "vector", 20:28:57 DEBUG "result": [ 20:28:57 DEBUG { 20:28:57 DEBUG "metric": { 20:28:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:28:57 DEBUG }, 20:28:57 DEBUG "value": [ 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG [...] 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:28:57 DEBUG }, 20:28:57 DEBUG "value": [ 20:28:57 DEBUG 1713641337, 20:28:57 DEBUG "0" 20:28:57 DEBUG ] 20:28:57 DEBUG } 20:28:57 DEBUG ] 20:28:57 DEBUG } 20:28:57 DEBUG } 20:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641337" 20:28:57 INFO [http_cmd]: http status code OK 20:28:57 DEBUG --- status code --- 20:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:57 DEBUG --- http response --- 20:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:28:57 DEBUG ----- output ----- 20:28:57 DEBUG { 20:28:57 DEBUG "status": "success", 20:28:57 DEBUG "data": { 20:28:57 DEBUG "resultType": "vector", 20:28:57 DEBUG "result": [ 20:28:57 DEBUG { 20:28:57 DEBUG "metric": { 20:28:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:28:57 DEBUG }, 20:28:57 DEBUG "value": [ 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG [...] 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG 20:28:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:28:57 DEBUG }, 20:28:57 DEBUG "value": [ 20:28:57 DEBUG 1713641337, 20:28:57 DEBUG "0" 20:28:57 DEBUG ] 20:28:57 DEBUG } 20:28:57 DEBUG ] 20:28:57 DEBUG } 20:28:57 DEBUG } 20:28:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641338" 20:28:58 INFO [http_cmd]: http status code OK 20:28:58 DEBUG --- status code --- 20:28:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:58 DEBUG --- http response --- 20:28:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:28:58 DEBUG ----- output ----- 20:28:58 DEBUG { 20:28:58 DEBUG "status": "success", 20:28:58 DEBUG "data": { 20:28:58 DEBUG "resultType": "vector", 20:28:58 DEBUG "result": [ 20:28:58 DEBUG { 20:28:58 DEBUG "metric": { 20:28:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:28:58 DEBUG }, 20:28:58 DEBUG "value": [ 20:28:58 DEBUG 20:28:58 DEBUG 20:28:58 DEBUG 20:28:58 DEBUG [...] 20:28:58 DEBUG 20:28:58 DEBUG 20:28:58 DEBUG 20:28:58 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:28:58 DEBUG }, 20:28:58 DEBUG "value": [ 20:28:58 DEBUG 1713641338, 20:28:58 DEBUG "0" 20:28:58 DEBUG ] 20:28:58 DEBUG } 20:28:58 DEBUG ] 20:28:58 DEBUG } 20:28:58 DEBUG } 20:28:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641338" 20:28:58 INFO [http_cmd]: http status code OK 20:28:58 DEBUG --- status code --- 20:28:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:58 DEBUG --- http response --- 20:28:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:28:58 DEBUG ----- output ----- 20:28:58 DEBUG { 20:28:58 DEBUG "status": "success", 20:28:58 DEBUG "data": { 20:28:58 DEBUG "resultType": "vector", 20:28:58 DEBUG "result": [ 20:28:58 DEBUG { 20:28:58 DEBUG "metric": { 20:28:58 DEBUG "pod": "ds-idrepo-0" 20:28:58 DEBUG }, 20:28:58 DEBUG "value": [ 20:28:58 DEBUG 20:28:58 DEBUG 20:28:58 DEBUG 20:28:58 DEBUG [...] 20:28:58 DEBUG 20:28:58 DEBUG 20:28:58 DEBUG 20:28:58 DEBUG "pod": "ds-idrepo-2" 20:28:58 DEBUG }, 20:28:58 DEBUG "value": [ 20:28:58 DEBUG 1713641338, 20:28:58 DEBUG "0" 20:28:58 DEBUG ] 20:28:58 DEBUG } 20:28:58 DEBUG ] 20:28:58 DEBUG } 20:28:58 DEBUG } 20:28:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641338" 20:28:58 INFO [http_cmd]: http status code OK 20:28:58 DEBUG --- status code --- 20:28:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:58 DEBUG --- http response --- 20:28:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:28:58 DEBUG ----- output ----- 20:28:58 DEBUG { 20:28:58 DEBUG "status": "success", 20:28:58 DEBUG "data": { 20:28:58 DEBUG "resultType": "vector", 20:28:58 DEBUG "result": [ 20:28:58 DEBUG { 20:28:58 DEBUG "metric": { 20:28:58 DEBUG "pod": "ds-cts-0" 20:28:58 DEBUG }, 20:28:58 DEBUG "value": [ 20:28:58 DEBUG 20:28:58 DEBUG 20:28:58 DEBUG 20:28:58 DEBUG [...] 20:28:58 DEBUG 20:28:58 DEBUG 20:28:58 DEBUG 20:28:58 DEBUG "pod": "ds-cts-2" 20:28:58 DEBUG }, 20:28:58 DEBUG "value": [ 20:28:58 DEBUG 1713641338, 20:28:58 DEBUG "0" 20:28:58 DEBUG ] 20:28:58 DEBUG } 20:28:58 DEBUG ] 20:28:58 DEBUG } 20:28:58 DEBUG } 20:28:59 INFO 20:28:59 INFO [loop_until]: kubectl --namespace=xlou top pods 20:28:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:28:59 INFO [loop_until]: OK (rc = 0) 20:28:59 DEBUG --- stdout --- 20:28:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 2747m 8624Mi am-767f6b67b5-qcqz2 2811m 9032Mi am-767f6b67b5-vbltz 3104m 8626Mi ds-cts-0 8194m 11008Mi ds-cts-1 7295m 11249Mi ds-cts-2 8247m 11053Mi ds-idrepo-0 897m 3896Mi ds-idrepo-1 822m 3888Mi ds-idrepo-2 801m 3838Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 4m 1239Mi lodemon-6f4955bbbb-clfwf 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 1039m 1537Mi 20:28:59 DEBUG --- stderr --- 20:28:59 DEBUG 20:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713641339" 20:28:59 INFO [http_cmd]: http status code OK 20:28:59 DEBUG --- status code --- 20:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:59 DEBUG --- http response --- 20:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:28:59 DEBUG ----- output ----- 20:28:59 DEBUG { 20:28:59 DEBUG "status": "success", 20:28:59 DEBUG "data": { 20:28:59 DEBUG "resultType": "vector", 20:28:59 DEBUG "result": [ 20:28:59 DEBUG { 20:28:59 DEBUG "metric": { 20:28:59 DEBUG "pod": "ds-idrepo-0" 20:28:59 DEBUG }, 20:28:59 DEBUG "value": [ 20:28:59 DEBUG 20:28:59 DEBUG 20:28:59 DEBUG 20:28:59 DEBUG [...] 20:28:59 DEBUG 20:28:59 DEBUG 20:28:59 DEBUG 20:28:59 DEBUG "pod": "ds-cts-2" 20:28:59 DEBUG }, 20:28:59 DEBUG "value": [ 20:28:59 DEBUG 1713641339, 20:28:59 DEBUG "0" 20:28:59 DEBUG ] 20:28:59 DEBUG } 20:28:59 DEBUG ] 20:28:59 DEBUG } 20:28:59 DEBUG } 20:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641339" 20:28:59 INFO [http_cmd]: http status code OK 20:28:59 DEBUG --- status code --- 20:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:59 DEBUG --- http response --- 20:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:28:59 DEBUG ----- output ----- 20:28:59 DEBUG { 20:28:59 DEBUG "status": "success", 20:28:59 DEBUG "data": { 20:28:59 DEBUG "resultType": "vector", 20:28:59 DEBUG "result": [ 20:28:59 DEBUG { 20:28:59 DEBUG "metric": { 20:28:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:28:59 DEBUG }, 20:28:59 DEBUG "value": [ 20:28:59 DEBUG 20:28:59 DEBUG 20:28:59 DEBUG 20:28:59 DEBUG [...] 20:28:59 DEBUG 20:28:59 DEBUG 20:28:59 DEBUG 20:28:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:28:59 DEBUG }, 20:28:59 DEBUG "value": [ 20:28:59 DEBUG 1713641339, 20:28:59 DEBUG "122772" 20:28:59 DEBUG ] 20:28:59 DEBUG } 20:28:59 DEBUG ] 20:28:59 DEBUG } 20:28:59 DEBUG } 20:28:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641339" 20:28:59 INFO [http_cmd]: http status code OK 20:28:59 DEBUG --- status code --- 20:28:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:28:59 DEBUG --- http response --- 20:28:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:28:59 DEBUG ----- output ----- 20:28:59 DEBUG { 20:28:59 DEBUG "status": "success", 20:28:59 DEBUG "data": { 20:28:59 DEBUG "resultType": "vector", 20:28:59 DEBUG "result": [ 20:28:59 DEBUG { 20:28:59 DEBUG "metric": { 20:28:59 DEBUG "instance": "10.142.0.107:9100" 20:28:59 DEBUG }, 20:28:59 DEBUG "value": [ 20:28:59 DEBUG 20:28:59 DEBUG 20:28:59 DEBUG 20:28:59 DEBUG [...] 20:28:59 DEBUG 20:28:59 DEBUG 20:28:59 DEBUG 20:28:59 DEBUG "instance": "10.142.0.43:9100" 20:28:59 DEBUG }, 20:28:59 DEBUG "value": [ 20:28:59 DEBUG 1713641339, 20:28:59 DEBUG "0.3252333333333875" 20:28:59 DEBUG ] 20:28:59 DEBUG } 20:28:59 DEBUG ] 20:28:59 DEBUG } 20:28:59 DEBUG } 20:29:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641340" 20:29:00 INFO [http_cmd]: http status code OK 20:29:00 DEBUG --- status code --- 20:29:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:00 DEBUG --- http response --- 20:29:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:29:00 DEBUG ----- output ----- 20:29:00 DEBUG { 20:29:00 DEBUG "status": "success", 20:29:00 DEBUG "data": { 20:29:00 DEBUG "resultType": "vector", 20:29:00 DEBUG "result": [ 20:29:00 DEBUG { 20:29:00 DEBUG "metric": { 20:29:00 DEBUG "instance": "10.142.0.107:9100" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG [...] 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG "instance": "10.142.0.43:9100" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 1713641340, 20:29:00 DEBUG "18553060.366666667" 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG } 20:29:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713641340" 20:29:00 INFO [http_cmd]: http status code OK 20:29:00 DEBUG --- status code --- 20:29:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:00 DEBUG --- http response --- 20:29:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:29:00 DEBUG ----- output ----- 20:29:00 DEBUG { 20:29:00 DEBUG "status": "success", 20:29:00 DEBUG "data": { 20:29:00 DEBUG "resultType": "vector", 20:29:00 DEBUG "result": [ 20:29:00 DEBUG { 20:29:00 DEBUG "metric": { 20:29:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG [...] 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 1713641340, 20:29:00 DEBUG "1359060992" 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG } 20:29:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641340" 20:29:00 INFO [http_cmd]: http status code OK 20:29:00 DEBUG --- status code --- 20:29:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:00 DEBUG --- http response --- 20:29:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:29:00 DEBUG ----- output ----- 20:29:00 DEBUG { 20:29:00 DEBUG "status": "success", 20:29:00 DEBUG "data": { 20:29:00 DEBUG "resultType": "vector", 20:29:00 DEBUG "result": [ 20:29:00 DEBUG { 20:29:00 DEBUG "metric": { 20:29:00 DEBUG "instance": "10.142.0.107:9100" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG [...] 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG "instance": "10.142.0.43:9100" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 1713641340, 20:29:00 DEBUG "18558943.133333333" 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG } 20:29:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641340" 20:29:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641340" 20:29:00 INFO [http_cmd]: http status code OK 20:29:00 DEBUG --- status code --- 20:29:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:00 DEBUG --- http response --- 20:29:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:29:00 DEBUG ----- output ----- 20:29:00 DEBUG { 20:29:00 DEBUG "status": "success", 20:29:00 DEBUG "data": { 20:29:00 DEBUG "resultType": "vector", 20:29:00 DEBUG "result": [ 20:29:00 DEBUG { 20:29:00 DEBUG "metric": { 20:29:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG [...] 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 1713641340, 20:29:00 DEBUG "0" 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG } 20:29:00 INFO [http_cmd]: http status code OK 20:29:00 DEBUG --- status code --- 20:29:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:00 DEBUG --- http response --- 20:29:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:29:00 DEBUG ----- output ----- 20:29:00 DEBUG { 20:29:00 DEBUG "status": "success", 20:29:00 DEBUG "data": { 20:29:00 DEBUG "resultType": "vector", 20:29:00 DEBUG "result": [ 20:29:00 DEBUG { 20:29:00 DEBUG "metric": { 20:29:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG [...] 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 1713641340, 20:29:00 DEBUG "52244444.623329796" 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG } 20:29:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641340" 20:29:00 INFO [http_cmd]: http status code OK 20:29:00 DEBUG --- status code --- 20:29:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:00 DEBUG --- http response --- 20:29:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:29:00 DEBUG ----- output ----- 20:29:00 DEBUG { 20:29:00 DEBUG "status": "success", 20:29:00 DEBUG "data": { 20:29:00 DEBUG "resultType": "vector", 20:29:00 DEBUG "result": [ 20:29:00 DEBUG { 20:29:00 DEBUG "metric": { 20:29:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG [...] 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 1713641340, 20:29:00 DEBUG "0" 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG } 20:29:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641340" 20:29:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713641340" 20:29:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641340" 20:29:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641340" 20:29:00 INFO [http_cmd]: http status code OK 20:29:00 INFO [http_cmd]: http status code OK 20:29:00 DEBUG --- status code --- 20:29:00 DEBUG --- status code --- 20:29:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:00 DEBUG --- http response --- 20:29:00 DEBUG --- http response --- 20:29:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:29:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:29:00 DEBUG ----- output ----- 20:29:00 DEBUG ----- output ----- 20:29:00 DEBUG { 20:29:00 DEBUG { 20:29:00 DEBUG "status": "success", 20:29:00 DEBUG "status": "success", 20:29:00 DEBUG "data": { 20:29:00 DEBUG "data": { 20:29:00 DEBUG "resultType": "vector", 20:29:00 DEBUG "resultType": "vector", 20:29:00 DEBUG "result": [ 20:29:00 DEBUG "result": [ 20:29:00 DEBUG { 20:29:00 DEBUG { 20:29:00 DEBUG "metric": { 20:29:00 DEBUG "metric": { 20:29:00 DEBUG "pod": "ds-cts-0" 20:29:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:29:00 DEBUG }, 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG [...] 20:29:00 DEBUG [...] 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG "pod": "ds-cts-2" 20:29:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:29:00 DEBUG }, 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 1713641340, 20:29:00 DEBUG 1713641340, 20:29:00 DEBUG "29.933333333333334" 20:29:00 DEBUG "0.159" 20:29:00 DEBUG ] 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG } 20:29:00 DEBUG ] 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG } 20:29:00 DEBUG } 20:29:00 DEBUG } 20:29:00 INFO [http_cmd]: http status code OK 20:29:00 DEBUG --- status code --- 20:29:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:00 DEBUG --- http response --- 20:29:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:29:00 DEBUG ----- output ----- 20:29:00 DEBUG { 20:29:00 DEBUG "status": "success", 20:29:00 DEBUG "data": { 20:29:00 DEBUG "resultType": "vector", 20:29:00 DEBUG "result": [ 20:29:00 DEBUG { 20:29:00 DEBUG "metric": { 20:29:00 DEBUG "pod": "ds-cts-0" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG [...] 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG "pod": "ds-cts-2" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 1713641340, 20:29:00 DEBUG "0" 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG } 20:29:00 INFO [http_cmd]: http status code OK 20:29:00 DEBUG --- status code --- 20:29:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:00 DEBUG --- http response --- 20:29:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:29:00 DEBUG ----- output ----- 20:29:00 DEBUG { 20:29:00 DEBUG "status": "success", 20:29:00 DEBUG "data": { 20:29:00 DEBUG "resultType": "vector", 20:29:00 DEBUG "result": [ 20:29:00 DEBUG { 20:29:00 DEBUG "metric": { 20:29:00 DEBUG "instance": "10.142.0.107:9100" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG [...] 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG 20:29:00 DEBUG "instance": "10.142.0.43:9100" 20:29:00 DEBUG }, 20:29:00 DEBUG "value": [ 20:29:00 DEBUG 1713641340, 20:29:00 DEBUG "0.16733333333333836" 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG ] 20:29:00 DEBUG } 20:29:00 DEBUG } 20:29:01 INFO 20:29:01 INFO [loop_until]: kubectl --namespace=xlou top node 20:29:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:29:01 INFO [loop_until]: OK (rc = 0) 20:29:01 DEBUG --- stdout --- 20:29:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 6214Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 871m 2% 7643Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2738m 9% 11377Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3367Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 8046m 26% 14238Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5194Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2855m 9% 11947Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 75m 0% 2943Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 3297m 11% 10859Mi 9% gke-xlou-cdm-ds-0f601751-j488 850m 2% 7229Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 7143m 23% 16331Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 902m 3% 7869Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 8191m 27% 13758Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 1077m 6% 7155Mi 12% 20:29:01 DEBUG --- stderr --- 20:29:01 DEBUG 20:29:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641395" 20:29:55 INFO [http_cmd]: http status code OK 20:29:55 DEBUG --- status code --- 20:29:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:55 DEBUG --- http response --- 20:29:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:29:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713641395" 20:29:55 ERROR [http_cmd]: ERROR 20:29:55 ERROR --- status code --- 20:29:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:29:55 ERROR --- http response --- 20:29:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:29:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641396" 20:29:56 INFO [http_cmd]: http status code OK 20:29:56 DEBUG --- status code --- 20:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:56 DEBUG --- http response --- 20:29:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:29:56 DEBUG ----- output ----- 20:29:56 DEBUG { 20:29:56 DEBUG "status": "success", 20:29:56 DEBUG "data": { 20:29:56 DEBUG "resultType": "vector", 20:29:56 DEBUG "result": [ 20:29:56 DEBUG { 20:29:56 DEBUG "metric": { 20:29:56 DEBUG "pod": "ds-idrepo-0" 20:29:56 DEBUG }, 20:29:56 DEBUG "value": [ 20:29:56 DEBUG 20:29:56 DEBUG 20:29:56 DEBUG 20:29:56 DEBUG [...] 20:29:56 DEBUG 20:29:56 DEBUG 20:29:56 DEBUG 20:29:56 DEBUG "pod": "ds-idrepo-2" 20:29:56 DEBUG }, 20:29:56 DEBUG "value": [ 20:29:56 DEBUG 1713641396, 20:29:56 DEBUG "0" 20:29:56 DEBUG ] 20:29:56 DEBUG } 20:29:56 DEBUG ] 20:29:56 DEBUG } 20:29:56 DEBUG } 20:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641396" 20:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641396" 20:29:56 INFO [http_cmd]: http status code OK 20:29:56 DEBUG --- status code --- 20:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:56 DEBUG --- http response --- 20:29:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:29:56 DEBUG ----- output ----- 20:29:56 DEBUG { 20:29:56 DEBUG "status": "success", 20:29:56 DEBUG "data": { 20:29:56 DEBUG "resultType": "vector", 20:29:56 DEBUG "result": [ 20:29:56 DEBUG { 20:29:56 DEBUG "metric": { 20:29:56 DEBUG "pod": "ds-idrepo-0" 20:29:56 DEBUG }, 20:29:56 DEBUG "value": [ 20:29:56 DEBUG 20:29:56 DEBUG 20:29:56 DEBUG 20:29:56 DEBUG [...] 20:29:56 DEBUG 20:29:56 DEBUG 20:29:56 DEBUG 20:29:56 DEBUG "pod": "ds-idrepo-2" 20:29:56 DEBUG }, 20:29:56 DEBUG "value": [ 20:29:56 DEBUG 1713641396, 20:29:56 DEBUG "0" 20:29:56 DEBUG ] 20:29:56 DEBUG } 20:29:56 DEBUG ] 20:29:56 DEBUG } 20:29:56 DEBUG } 20:29:56 INFO [http_cmd]: http status code OK 20:29:56 DEBUG --- status code --- 20:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:56 DEBUG --- http response --- 20:29:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:29:56 DEBUG ----- output ----- 20:29:56 DEBUG { 20:29:56 DEBUG "status": "success", 20:29:56 DEBUG "data": { 20:29:56 DEBUG "resultType": "vector", 20:29:56 DEBUG "result": [ 20:29:56 DEBUG { 20:29:56 DEBUG "metric": { 20:29:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:29:56 DEBUG }, 20:29:56 DEBUG "value": [ 20:29:56 DEBUG 20:29:56 DEBUG 20:29:56 DEBUG 20:29:56 DEBUG [...] 20:29:56 DEBUG 20:29:56 DEBUG 20:29:56 DEBUG 20:29:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:29:56 DEBUG }, 20:29:56 DEBUG "value": [ 20:29:56 DEBUG 1713641396, 20:29:56 DEBUG "0" 20:29:56 DEBUG ] 20:29:56 DEBUG } 20:29:56 DEBUG ] 20:29:56 DEBUG } 20:29:56 DEBUG } 20:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641397" 20:29:57 INFO [http_cmd]: http status code OK 20:29:57 DEBUG --- status code --- 20:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:57 DEBUG --- http response --- 20:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:29:57 DEBUG ----- output ----- 20:29:57 DEBUG { 20:29:57 DEBUG "status": "success", 20:29:57 DEBUG "data": { 20:29:57 DEBUG "resultType": "vector", 20:29:57 DEBUG "result": [ 20:29:57 DEBUG { 20:29:57 DEBUG "metric": { 20:29:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:29:57 DEBUG }, 20:29:57 DEBUG "value": [ 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG [...] 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG "pod": "idm-b85f9d978-txpwv" 20:29:57 DEBUG }, 20:29:57 DEBUG "value": [ 20:29:57 DEBUG 1713641397, 20:29:57 DEBUG "0" 20:29:57 DEBUG ] 20:29:57 DEBUG } 20:29:57 DEBUG ] 20:29:57 DEBUG } 20:29:57 DEBUG } 20:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641397" 20:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641397" 20:29:57 INFO [http_cmd]: http status code OK 20:29:57 DEBUG --- status code --- 20:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:57 DEBUG --- http response --- 20:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:29:57 DEBUG ----- output ----- 20:29:57 DEBUG { 20:29:57 DEBUG "status": "success", 20:29:57 DEBUG "data": { 20:29:57 DEBUG "resultType": "vector", 20:29:57 DEBUG "result": [ 20:29:57 DEBUG { 20:29:57 DEBUG "metric": { 20:29:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:29:57 DEBUG }, 20:29:57 DEBUG "value": [ 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG [...] 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:29:57 DEBUG }, 20:29:57 DEBUG "value": [ 20:29:57 DEBUG 1713641397, 20:29:57 DEBUG "0" 20:29:57 DEBUG ] 20:29:57 DEBUG } 20:29:57 DEBUG ] 20:29:57 DEBUG } 20:29:57 DEBUG } 20:29:57 INFO [http_cmd]: http status code OK 20:29:57 DEBUG --- status code --- 20:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:57 DEBUG --- http response --- 20:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:29:57 DEBUG ----- output ----- 20:29:57 DEBUG { 20:29:57 DEBUG "status": "success", 20:29:57 DEBUG "data": { 20:29:57 DEBUG "resultType": "vector", 20:29:57 DEBUG "result": [ 20:29:57 DEBUG { 20:29:57 DEBUG "metric": { 20:29:57 DEBUG "instance": "10.142.0.107:9100" 20:29:57 DEBUG }, 20:29:57 DEBUG "value": [ 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG [...] 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG "instance": "10.142.0.43:9100" 20:29:57 DEBUG }, 20:29:57 DEBUG "value": [ 20:29:57 DEBUG 1713641397, 20:29:57 DEBUG "3764006550528" 20:29:57 DEBUG ] 20:29:57 DEBUG } 20:29:57 DEBUG ] 20:29:57 DEBUG } 20:29:57 DEBUG } 20:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713641397" 20:29:57 INFO [http_cmd]: http status code OK 20:29:57 DEBUG --- status code --- 20:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:57 DEBUG --- http response --- 20:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:29:57 DEBUG ----- output ----- 20:29:57 DEBUG { 20:29:57 DEBUG "status": "success", 20:29:57 DEBUG "data": { 20:29:57 DEBUG "resultType": "vector", 20:29:57 DEBUG "result": [ 20:29:57 DEBUG { 20:29:57 DEBUG "metric": { 20:29:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:29:57 DEBUG }, 20:29:57 DEBUG "value": [ 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG [...] 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:29:57 DEBUG }, 20:29:57 DEBUG "value": [ 20:29:57 DEBUG 1713641397, 20:29:57 DEBUG "1.0327156511005742" 20:29:57 DEBUG ] 20:29:57 DEBUG } 20:29:57 DEBUG ] 20:29:57 DEBUG } 20:29:57 DEBUG } 20:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641397" 20:29:57 INFO [http_cmd]: http status code OK 20:29:57 DEBUG --- status code --- 20:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:57 DEBUG --- http response --- 20:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:29:57 DEBUG ----- output ----- 20:29:57 DEBUG { 20:29:57 DEBUG "status": "success", 20:29:57 DEBUG "data": { 20:29:57 DEBUG "resultType": "vector", 20:29:57 DEBUG "result": [ 20:29:57 DEBUG { 20:29:57 DEBUG "metric": { 20:29:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:29:57 DEBUG }, 20:29:57 DEBUG "value": [ 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG [...] 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:29:57 DEBUG }, 20:29:57 DEBUG "value": [ 20:29:57 DEBUG 1713641397, 20:29:57 DEBUG "0" 20:29:57 DEBUG ] 20:29:57 DEBUG } 20:29:57 DEBUG ] 20:29:57 DEBUG } 20:29:57 DEBUG } 20:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641397" 20:29:57 INFO [http_cmd]: http status code OK 20:29:57 DEBUG --- status code --- 20:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:57 DEBUG --- http response --- 20:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:29:57 DEBUG ----- output ----- 20:29:57 DEBUG { 20:29:57 DEBUG "status": "success", 20:29:57 DEBUG "data": { 20:29:57 DEBUG "resultType": "vector", 20:29:57 DEBUG "result": [ 20:29:57 DEBUG { 20:29:57 DEBUG "metric": { 20:29:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:29:57 DEBUG }, 20:29:57 DEBUG "value": [ 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG [...] 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG 20:29:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:29:57 DEBUG }, 20:29:57 DEBUG "value": [ 20:29:57 DEBUG 1713641397, 20:29:57 DEBUG "0" 20:29:57 DEBUG ] 20:29:57 DEBUG } 20:29:57 DEBUG ] 20:29:57 DEBUG } 20:29:57 DEBUG } 20:29:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641398" 20:29:58 INFO [http_cmd]: http status code OK 20:29:58 DEBUG --- status code --- 20:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:58 DEBUG --- http response --- 20:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:29:58 DEBUG ----- output ----- 20:29:58 DEBUG { 20:29:58 DEBUG "status": "success", 20:29:58 DEBUG "data": { 20:29:58 DEBUG "resultType": "vector", 20:29:58 DEBUG "result": [ 20:29:58 DEBUG { 20:29:58 DEBUG "metric": { 20:29:58 DEBUG "pod": "ds-idrepo-0" 20:29:58 DEBUG }, 20:29:58 DEBUG "value": [ 20:29:58 DEBUG 20:29:58 DEBUG 20:29:58 DEBUG 20:29:58 DEBUG [...] 20:29:58 DEBUG 20:29:58 DEBUG 20:29:58 DEBUG 20:29:58 DEBUG "pod": "idm-b85f9d978-txpwv" 20:29:58 DEBUG }, 20:29:58 DEBUG "value": [ 20:29:58 DEBUG 1713641398, 20:29:58 DEBUG "0.0019544216238976723" 20:29:58 DEBUG ] 20:29:58 DEBUG } 20:29:58 DEBUG ] 20:29:58 DEBUG } 20:29:58 DEBUG } 20:29:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641398" 20:29:58 INFO [http_cmd]: http status code OK 20:29:58 DEBUG --- status code --- 20:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:58 DEBUG --- http response --- 20:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:29:58 DEBUG ----- output ----- 20:29:58 DEBUG { 20:29:58 DEBUG "status": "success", 20:29:58 DEBUG "data": { 20:29:58 DEBUG "resultType": "vector", 20:29:58 DEBUG "result": [ 20:29:58 DEBUG { 20:29:58 DEBUG "metric": { 20:29:58 DEBUG "pod": "ds-idrepo-0" 20:29:58 DEBUG }, 20:29:58 DEBUG "value": [ 20:29:58 DEBUG 20:29:58 DEBUG 20:29:58 DEBUG 20:29:58 DEBUG [...] 20:29:58 DEBUG 20:29:58 DEBUG 20:29:58 DEBUG 20:29:58 DEBUG "pod": "ds-idrepo-2" 20:29:58 DEBUG }, 20:29:58 DEBUG "value": [ 20:29:58 DEBUG 1713641398, 20:29:58 DEBUG "0" 20:29:58 DEBUG ] 20:29:58 DEBUG } 20:29:58 DEBUG ] 20:29:58 DEBUG } 20:29:58 DEBUG } 20:29:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641398" 20:29:58 INFO [http_cmd]: http status code OK 20:29:58 DEBUG --- status code --- 20:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:58 DEBUG --- http response --- 20:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:29:58 DEBUG ----- output ----- 20:29:58 DEBUG { 20:29:58 DEBUG "status": "success", 20:29:58 DEBUG "data": { 20:29:58 DEBUG "resultType": "vector", 20:29:58 DEBUG "result": [ 20:29:58 DEBUG { 20:29:58 DEBUG "metric": { 20:29:58 DEBUG "pod": "ds-cts-0" 20:29:58 DEBUG }, 20:29:58 DEBUG "value": [ 20:29:58 DEBUG 20:29:58 DEBUG 20:29:58 DEBUG 20:29:58 DEBUG [...] 20:29:58 DEBUG 20:29:58 DEBUG 20:29:58 DEBUG 20:29:58 DEBUG "pod": "ds-cts-2" 20:29:58 DEBUG }, 20:29:58 DEBUG "value": [ 20:29:58 DEBUG 1713641398, 20:29:58 DEBUG "0" 20:29:58 DEBUG ] 20:29:58 DEBUG } 20:29:58 DEBUG ] 20:29:58 DEBUG } 20:29:58 DEBUG } 20:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713641399" 20:29:59 INFO [http_cmd]: http status code OK 20:29:59 DEBUG --- status code --- 20:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:59 DEBUG --- http response --- 20:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:29:59 DEBUG ----- output ----- 20:29:59 DEBUG { 20:29:59 DEBUG "status": "success", 20:29:59 DEBUG "data": { 20:29:59 DEBUG "resultType": "vector", 20:29:59 DEBUG "result": [ 20:29:59 DEBUG { 20:29:59 DEBUG "metric": { 20:29:59 DEBUG "pod": "ds-idrepo-0" 20:29:59 DEBUG }, 20:29:59 DEBUG "value": [ 20:29:59 DEBUG 20:29:59 DEBUG 20:29:59 DEBUG 20:29:59 DEBUG [...] 20:29:59 DEBUG 20:29:59 DEBUG 20:29:59 DEBUG 20:29:59 DEBUG "pod": "ds-cts-2" 20:29:59 DEBUG }, 20:29:59 DEBUG "value": [ 20:29:59 DEBUG 1713641399, 20:29:59 DEBUG "0" 20:29:59 DEBUG ] 20:29:59 DEBUG } 20:29:59 DEBUG ] 20:29:59 DEBUG } 20:29:59 DEBUG } 20:29:59 INFO 20:29:59 INFO [loop_until]: kubectl --namespace=xlou top pods 20:29:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:29:59 INFO [loop_until]: OK (rc = 0) 20:29:59 DEBUG --- stdout --- 20:29:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 6m 8615Mi am-767f6b67b5-qcqz2 611m 9027Mi am-767f6b67b5-vbltz 6m 8622Mi ds-cts-0 14m 11010Mi ds-cts-1 15m 10804Mi ds-cts-2 1634m 11055Mi ds-idrepo-0 114m 3895Mi ds-idrepo-1 12m 3879Mi ds-idrepo-2 13m 3836Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 851m 1995Mi 20:29:59 DEBUG --- stderr --- 20:29:59 DEBUG 20:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641399" 20:29:59 INFO [http_cmd]: http status code OK 20:29:59 DEBUG --- status code --- 20:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:59 DEBUG --- http response --- 20:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:29:59 DEBUG ----- output ----- 20:29:59 DEBUG { 20:29:59 DEBUG "status": "success", 20:29:59 DEBUG "data": { 20:29:59 DEBUG "resultType": "vector", 20:29:59 DEBUG "result": [ 20:29:59 DEBUG { 20:29:59 DEBUG "metric": { 20:29:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:29:59 DEBUG }, 20:29:59 DEBUG "value": [ 20:29:59 DEBUG 20:29:59 DEBUG 20:29:59 DEBUG 20:29:59 DEBUG [...] 20:29:59 DEBUG 20:29:59 DEBUG 20:29:59 DEBUG 20:29:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:29:59 DEBUG }, 20:29:59 DEBUG "value": [ 20:29:59 DEBUG 1713641399, 20:29:59 DEBUG "1" 20:29:59 DEBUG ] 20:29:59 DEBUG } 20:29:59 DEBUG ] 20:29:59 DEBUG } 20:29:59 DEBUG } 20:29:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641399" 20:29:59 INFO [http_cmd]: http status code OK 20:29:59 DEBUG --- status code --- 20:29:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:29:59 DEBUG --- http response --- 20:29:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:29:59 DEBUG ----- output ----- 20:29:59 DEBUG { 20:29:59 DEBUG "status": "success", 20:29:59 DEBUG "data": { 20:29:59 DEBUG "resultType": "vector", 20:29:59 DEBUG "result": [ 20:29:59 DEBUG { 20:29:59 DEBUG "metric": { 20:29:59 DEBUG "instance": "10.142.0.107:9100" 20:29:59 DEBUG }, 20:29:59 DEBUG "value": [ 20:29:59 DEBUG 20:29:59 DEBUG 20:29:59 DEBUG 20:29:59 DEBUG [...] 20:29:59 DEBUG 20:29:59 DEBUG 20:29:59 DEBUG 20:29:59 DEBUG "instance": "10.142.0.43:9100" 20:29:59 DEBUG }, 20:29:59 DEBUG "value": [ 20:29:59 DEBUG 1713641399, 20:29:59 DEBUG "0.0602333333333822" 20:29:59 DEBUG ] 20:29:59 DEBUG } 20:29:59 DEBUG ] 20:29:59 DEBUG } 20:29:59 DEBUG } 20:30:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641400" 20:30:00 INFO [http_cmd]: http status code OK 20:30:00 DEBUG --- status code --- 20:30:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:00 DEBUG --- http response --- 20:30:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:30:00 DEBUG ----- output ----- 20:30:00 DEBUG { 20:30:00 DEBUG "status": "success", 20:30:00 DEBUG "data": { 20:30:00 DEBUG "resultType": "vector", 20:30:00 DEBUG "result": [ 20:30:00 DEBUG { 20:30:00 DEBUG "metric": { 20:30:00 DEBUG "instance": "10.142.0.107:9100" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG [...] 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG "instance": "10.142.0.43:9100" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 1713641400, 20:30:00 DEBUG "6776003.766666666" 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG } 20:30:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713641400" 20:30:00 INFO [http_cmd]: http status code OK 20:30:00 DEBUG --- status code --- 20:30:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:00 DEBUG --- http response --- 20:30:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:30:00 DEBUG ----- output ----- 20:30:00 DEBUG { 20:30:00 DEBUG "status": "success", 20:30:00 DEBUG "data": { 20:30:00 DEBUG "resultType": "vector", 20:30:00 DEBUG "result": [ 20:30:00 DEBUG { 20:30:00 DEBUG "metric": { 20:30:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG [...] 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 1713641400, 20:30:00 DEBUG "2253651968" 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG } 20:30:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641400" 20:30:00 INFO [http_cmd]: http status code OK 20:30:00 DEBUG --- status code --- 20:30:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:00 DEBUG --- http response --- 20:30:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:30:00 DEBUG ----- output ----- 20:30:00 DEBUG { 20:30:00 DEBUG "status": "success", 20:30:00 DEBUG "data": { 20:30:00 DEBUG "resultType": "vector", 20:30:00 DEBUG "result": [ 20:30:00 DEBUG { 20:30:00 DEBUG "metric": { 20:30:00 DEBUG "instance": "10.142.0.107:9100" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG [...] 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG "instance": "10.142.0.43:9100" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 1713641400, 20:30:00 DEBUG "6781524.666666666" 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG } 20:30:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641400" 20:30:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641400" 20:30:00 INFO [http_cmd]: http status code OK 20:30:00 DEBUG --- status code --- 20:30:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:00 DEBUG --- http response --- 20:30:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:30:00 DEBUG ----- output ----- 20:30:00 DEBUG { 20:30:00 DEBUG "status": "success", 20:30:00 DEBUG "data": { 20:30:00 DEBUG "resultType": "vector", 20:30:00 DEBUG "result": [ 20:30:00 DEBUG { 20:30:00 DEBUG "metric": { 20:30:00 DEBUG "pod": "ds-idrepo-0" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG [...] 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG "pod": "idm-b85f9d978-txpwv" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 1713641400, 20:30:00 DEBUG "38055203.96421259" 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG } 20:30:00 INFO [http_cmd]: http status code OK 20:30:00 DEBUG --- status code --- 20:30:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:00 DEBUG --- http response --- 20:30:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:30:00 DEBUG ----- output ----- 20:30:00 DEBUG { 20:30:00 DEBUG "status": "success", 20:30:00 DEBUG "data": { 20:30:00 DEBUG "resultType": "vector", 20:30:00 DEBUG "result": [ 20:30:00 DEBUG { 20:30:00 DEBUG "metric": { 20:30:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG [...] 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 1713641400, 20:30:00 DEBUG "0" 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG } 20:30:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641400" 20:30:00 INFO [http_cmd]: http status code OK 20:30:00 DEBUG --- status code --- 20:30:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:00 DEBUG --- http response --- 20:30:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 20:30:00 DEBUG ----- output ----- 20:30:00 DEBUG { 20:30:00 DEBUG "status": "success", 20:30:00 DEBUG "data": { 20:30:00 DEBUG "resultType": "vector", 20:30:00 DEBUG "result": [ 20:30:00 DEBUG { 20:30:00 DEBUG "metric": { 20:30:00 DEBUG "pod": "ds-idrepo-0" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG [...] 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG "pod": "idm-b85f9d978-txpwv" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 1713641400, 20:30:00 DEBUG "0" 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG } 20:30:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641400" 20:30:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641400" 20:30:00 INFO [http_cmd]: http status code OK 20:30:00 DEBUG --- status code --- 20:30:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:00 DEBUG --- http response --- 20:30:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:30:00 DEBUG ----- output ----- 20:30:00 DEBUG { 20:30:00 DEBUG "status": "success", 20:30:00 DEBUG "data": { 20:30:00 DEBUG "resultType": "vector", 20:30:00 DEBUG "result": [ 20:30:00 DEBUG { 20:30:00 DEBUG "metric": { 20:30:00 DEBUG "pod": "ds-cts-0" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG [...] 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG "pod": "ds-cts-2" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 1713641400, 20:30:00 DEBUG "0" 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG } 20:30:00 INFO [http_cmd]: http status code OK 20:30:00 DEBUG --- status code --- 20:30:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:00 DEBUG --- http response --- 20:30:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:30:00 DEBUG ----- output ----- 20:30:00 DEBUG { 20:30:00 DEBUG "status": "success", 20:30:00 DEBUG "data": { 20:30:00 DEBUG "resultType": "vector", 20:30:00 DEBUG "result": [ 20:30:00 DEBUG { 20:30:00 DEBUG "metric": { 20:30:00 DEBUG "instance": "10.142.0.107:9100" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG [...] 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG "instance": "10.142.0.43:9100" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 1713641400, 20:30:00 DEBUG "0.03166666666665871" 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG } 20:30:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641400" 20:30:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713641400" 20:30:00 INFO [http_cmd]: http status code OK 20:30:00 INFO [http_cmd]: http status code OK 20:30:00 DEBUG --- status code --- 20:30:00 DEBUG --- status code --- 20:30:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:00 DEBUG --- http response --- 20:30:00 DEBUG --- http response --- 20:30:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:30:00 DEBUG ----- output ----- 20:30:00 DEBUG { 20:30:00 DEBUG "status": "success", 20:30:00 DEBUG "data": { 20:30:00 DEBUG "resultType": "vector", 20:30:00 DEBUG "result": [ 20:30:00 DEBUG { 20:30:00 DEBUG "metric": { 20:30:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG [...] 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 1713641400, 20:30:00 DEBUG "0.159" 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG } 20:30:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:30:00 DEBUG ----- output ----- 20:30:00 DEBUG { 20:30:00 DEBUG "status": "success", 20:30:00 DEBUG "data": { 20:30:00 DEBUG "resultType": "vector", 20:30:00 DEBUG "result": [ 20:30:00 DEBUG { 20:30:00 DEBUG "metric": { 20:30:00 DEBUG "pod": "ds-cts-0" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG [...] 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG 20:30:00 DEBUG "pod": "ds-cts-2" 20:30:00 DEBUG }, 20:30:00 DEBUG "value": [ 20:30:00 DEBUG 1713641400, 20:30:00 DEBUG "30.03333333333333" 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG ] 20:30:00 DEBUG } 20:30:00 DEBUG } 20:30:01 INFO 20:30:01 INFO [loop_until]: kubectl --namespace=xlou top node 20:30:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:30:01 INFO [loop_until]: OK (rc = 0) 20:30:01 DEBUG --- stdout --- 20:30:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6203Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 70m 0% 7627Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 55m 0% 11357Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3364Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 73m 0% 14215Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 137m 0% 5194Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 11948Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 75m 0% 2937Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 59m 0% 10877Mi 9% gke-xlou-cdm-ds-0f601751-j488 69m 0% 7220Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 78m 0% 15854Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 86m 0% 7851Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 614m 2% 13744Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 810m 5% 7549Mi 12% 20:30:01 DEBUG --- stderr --- 20:30:01 DEBUG 20:30:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641455" 20:30:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713641455" 20:30:55 INFO [http_cmd]: http status code OK 20:30:55 DEBUG --- status code --- 20:30:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:55 DEBUG --- http response --- 20:30:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:30:55 ERROR [http_cmd]: ERROR 20:30:55 ERROR --- status code --- 20:30:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:30:55 ERROR --- http response --- 20:30:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:30:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641456" 20:30:56 INFO [http_cmd]: http status code OK 20:30:56 DEBUG --- status code --- 20:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:56 DEBUG --- http response --- 20:30:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:30:56 DEBUG ----- output ----- 20:30:56 DEBUG { 20:30:56 DEBUG "status": "success", 20:30:56 DEBUG "data": { 20:30:56 DEBUG "resultType": "vector", 20:30:56 DEBUG "result": [ 20:30:56 DEBUG { 20:30:56 DEBUG "metric": { 20:30:56 DEBUG "pod": "ds-idrepo-0" 20:30:56 DEBUG }, 20:30:56 DEBUG "value": [ 20:30:56 DEBUG 20:30:56 DEBUG 20:30:56 DEBUG 20:30:56 DEBUG [...] 20:30:56 DEBUG 20:30:56 DEBUG 20:30:56 DEBUG 20:30:56 DEBUG "pod": "ds-idrepo-2" 20:30:56 DEBUG }, 20:30:56 DEBUG "value": [ 20:30:56 DEBUG 1713641456, 20:30:56 DEBUG "0" 20:30:56 DEBUG ] 20:30:56 DEBUG } 20:30:56 DEBUG ] 20:30:56 DEBUG } 20:30:56 DEBUG } 20:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641456" 20:30:56 INFO [http_cmd]: http status code OK 20:30:56 DEBUG --- status code --- 20:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:56 DEBUG --- http response --- 20:30:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:30:56 DEBUG ----- output ----- 20:30:56 DEBUG { 20:30:56 DEBUG "status": "success", 20:30:56 DEBUG "data": { 20:30:56 DEBUG "resultType": "vector", 20:30:56 DEBUG "result": [ 20:30:56 DEBUG { 20:30:56 DEBUG "metric": { 20:30:56 DEBUG "pod": "ds-idrepo-0" 20:30:56 DEBUG }, 20:30:56 DEBUG "value": [ 20:30:56 DEBUG 20:30:56 DEBUG 20:30:56 DEBUG 20:30:56 DEBUG [...] 20:30:56 DEBUG 20:30:56 DEBUG 20:30:56 DEBUG 20:30:56 DEBUG "pod": "ds-idrepo-2" 20:30:56 DEBUG }, 20:30:56 DEBUG "value": [ 20:30:56 DEBUG 1713641456, 20:30:56 DEBUG "0" 20:30:56 DEBUG ] 20:30:56 DEBUG } 20:30:56 DEBUG ] 20:30:56 DEBUG } 20:30:56 DEBUG } 20:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641456" 20:30:56 INFO [http_cmd]: http status code OK 20:30:56 DEBUG --- status code --- 20:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:56 DEBUG --- http response --- 20:30:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:30:56 DEBUG ----- output ----- 20:30:56 DEBUG { 20:30:56 DEBUG "status": "success", 20:30:56 DEBUG "data": { 20:30:56 DEBUG "resultType": "vector", 20:30:56 DEBUG "result": [ 20:30:56 DEBUG { 20:30:56 DEBUG "metric": { 20:30:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:30:56 DEBUG }, 20:30:56 DEBUG "value": [ 20:30:56 DEBUG 20:30:56 DEBUG 20:30:56 DEBUG 20:30:56 DEBUG [...] 20:30:56 DEBUG 20:30:56 DEBUG 20:30:56 DEBUG 20:30:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:30:56 DEBUG }, 20:30:56 DEBUG "value": [ 20:30:56 DEBUG 1713641456, 20:30:56 DEBUG "0" 20:30:56 DEBUG ] 20:30:56 DEBUG } 20:30:56 DEBUG ] 20:30:56 DEBUG } 20:30:56 DEBUG } 20:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641457" 20:30:57 INFO [http_cmd]: http status code OK 20:30:57 DEBUG --- status code --- 20:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:57 DEBUG --- http response --- 20:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:30:57 DEBUG ----- output ----- 20:30:57 DEBUG { 20:30:57 DEBUG "status": "success", 20:30:57 DEBUG "data": { 20:30:57 DEBUG "resultType": "vector", 20:30:57 DEBUG "result": [ 20:30:57 DEBUG { 20:30:57 DEBUG "metric": { 20:30:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:30:57 DEBUG }, 20:30:57 DEBUG "value": [ 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG [...] 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG "pod": "idm-b85f9d978-wnhxb" 20:30:57 DEBUG }, 20:30:57 DEBUG "value": [ 20:30:57 DEBUG 1713641457, 20:30:57 DEBUG "1152.782869538807" 20:30:57 DEBUG ] 20:30:57 DEBUG } 20:30:57 DEBUG ] 20:30:57 DEBUG } 20:30:57 DEBUG } 20:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641457" 20:30:57 INFO [http_cmd]: http status code OK 20:30:57 DEBUG --- status code --- 20:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:57 DEBUG --- http response --- 20:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:30:57 DEBUG ----- output ----- 20:30:57 DEBUG { 20:30:57 DEBUG "status": "success", 20:30:57 DEBUG "data": { 20:30:57 DEBUG "resultType": "vector", 20:30:57 DEBUG "result": [ 20:30:57 DEBUG { 20:30:57 DEBUG "metric": { 20:30:57 DEBUG "instance": "10.142.0.107:9100" 20:30:57 DEBUG }, 20:30:57 DEBUG "value": [ 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG [...] 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG "instance": "10.142.0.43:9100" 20:30:57 DEBUG }, 20:30:57 DEBUG "value": [ 20:30:57 DEBUG 1713641457, 20:30:57 DEBUG "3764101139456" 20:30:57 DEBUG ] 20:30:57 DEBUG } 20:30:57 DEBUG ] 20:30:57 DEBUG } 20:30:57 DEBUG } 20:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641457" 20:30:57 INFO [http_cmd]: http status code OK 20:30:57 DEBUG --- status code --- 20:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:57 DEBUG --- http response --- 20:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:30:57 DEBUG ----- output ----- 20:30:57 DEBUG { 20:30:57 DEBUG "status": "success", 20:30:57 DEBUG "data": { 20:30:57 DEBUG "resultType": "vector", 20:30:57 DEBUG "result": [ 20:30:57 DEBUG { 20:30:57 DEBUG "metric": { 20:30:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:30:57 DEBUG }, 20:30:57 DEBUG "value": [ 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG [...] 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:30:57 DEBUG }, 20:30:57 DEBUG "value": [ 20:30:57 DEBUG 1713641457, 20:30:57 DEBUG "0" 20:30:57 DEBUG ] 20:30:57 DEBUG } 20:30:57 DEBUG ] 20:30:57 DEBUG } 20:30:57 DEBUG } 20:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713641457" 20:30:57 INFO [http_cmd]: http status code OK 20:30:57 DEBUG --- status code --- 20:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:57 DEBUG --- http response --- 20:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:30:57 DEBUG ----- output ----- 20:30:57 DEBUG { 20:30:57 DEBUG "status": "success", 20:30:57 DEBUG "data": { 20:30:57 DEBUG "resultType": "vector", 20:30:57 DEBUG "result": [ 20:30:57 DEBUG { 20:30:57 DEBUG "metric": { 20:30:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:30:57 DEBUG }, 20:30:57 DEBUG "value": [ 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG [...] 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:30:57 DEBUG }, 20:30:57 DEBUG "value": [ 20:30:57 DEBUG 1713641457, 20:30:57 DEBUG "0.7526147469934679" 20:30:57 DEBUG ] 20:30:57 DEBUG } 20:30:57 DEBUG ] 20:30:57 DEBUG } 20:30:57 DEBUG } 20:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641457" 20:30:57 INFO [http_cmd]: http status code OK 20:30:57 DEBUG --- status code --- 20:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:57 DEBUG --- http response --- 20:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:30:57 DEBUG ----- output ----- 20:30:57 DEBUG { 20:30:57 DEBUG "status": "success", 20:30:57 DEBUG "data": { 20:30:57 DEBUG "resultType": "vector", 20:30:57 DEBUG "result": [ 20:30:57 DEBUG { 20:30:57 DEBUG "metric": { 20:30:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:30:57 DEBUG }, 20:30:57 DEBUG "value": [ 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG [...] 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:30:57 DEBUG }, 20:30:57 DEBUG "value": [ 20:30:57 DEBUG 1713641457, 20:30:57 DEBUG "0" 20:30:57 DEBUG ] 20:30:57 DEBUG } 20:30:57 DEBUG ] 20:30:57 DEBUG } 20:30:57 DEBUG } 20:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641457" 20:30:57 INFO [http_cmd]: http status code OK 20:30:57 DEBUG --- status code --- 20:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:57 DEBUG --- http response --- 20:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:30:57 DEBUG ----- output ----- 20:30:57 DEBUG { 20:30:57 DEBUG "status": "success", 20:30:57 DEBUG "data": { 20:30:57 DEBUG "resultType": "vector", 20:30:57 DEBUG "result": [ 20:30:57 DEBUG { 20:30:57 DEBUG "metric": { 20:30:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:30:57 DEBUG }, 20:30:57 DEBUG "value": [ 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG [...] 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG 20:30:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:30:57 DEBUG }, 20:30:57 DEBUG "value": [ 20:30:57 DEBUG 1713641457, 20:30:57 DEBUG "0" 20:30:57 DEBUG ] 20:30:57 DEBUG } 20:30:57 DEBUG ] 20:30:57 DEBUG } 20:30:57 DEBUG } 20:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641458" 20:30:58 INFO [http_cmd]: http status code OK 20:30:58 DEBUG --- status code --- 20:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:58 DEBUG --- http response --- 20:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:30:58 DEBUG ----- output ----- 20:30:58 DEBUG { 20:30:58 DEBUG "status": "success", 20:30:58 DEBUG "data": { 20:30:58 DEBUG "resultType": "vector", 20:30:58 DEBUG "result": [ 20:30:58 DEBUG { 20:30:58 DEBUG "metric": { 20:30:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:30:58 DEBUG }, 20:30:58 DEBUG "value": [ 20:30:58 DEBUG 20:30:58 DEBUG 20:30:58 DEBUG 20:30:58 DEBUG [...] 20:30:58 DEBUG 20:30:58 DEBUG 20:30:58 DEBUG 20:30:58 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:30:58 DEBUG }, 20:30:58 DEBUG "value": [ 20:30:58 DEBUG 1713641458, 20:30:58 DEBUG "0" 20:30:58 DEBUG ] 20:30:58 DEBUG } 20:30:58 DEBUG ] 20:30:58 DEBUG } 20:30:58 DEBUG } 20:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641458" 20:30:58 INFO [http_cmd]: http status code OK 20:30:58 DEBUG --- status code --- 20:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:58 DEBUG --- http response --- 20:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:30:58 DEBUG ----- output ----- 20:30:58 DEBUG { 20:30:58 DEBUG "status": "success", 20:30:58 DEBUG "data": { 20:30:58 DEBUG "resultType": "vector", 20:30:58 DEBUG "result": [ 20:30:58 DEBUG { 20:30:58 DEBUG "metric": { 20:30:58 DEBUG "pod": "ds-idrepo-0" 20:30:58 DEBUG }, 20:30:58 DEBUG "value": [ 20:30:58 DEBUG 20:30:58 DEBUG 20:30:58 DEBUG 20:30:58 DEBUG [...] 20:30:58 DEBUG 20:30:58 DEBUG 20:30:58 DEBUG 20:30:58 DEBUG "pod": "ds-idrepo-2" 20:30:58 DEBUG }, 20:30:58 DEBUG "value": [ 20:30:58 DEBUG 1713641458, 20:30:58 DEBUG "0" 20:30:58 DEBUG ] 20:30:58 DEBUG } 20:30:58 DEBUG ] 20:30:58 DEBUG } 20:30:58 DEBUG } 20:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641458" 20:30:58 INFO [http_cmd]: http status code OK 20:30:58 DEBUG --- status code --- 20:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:58 DEBUG --- http response --- 20:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:30:58 DEBUG ----- output ----- 20:30:58 DEBUG { 20:30:58 DEBUG "status": "success", 20:30:58 DEBUG "data": { 20:30:58 DEBUG "resultType": "vector", 20:30:58 DEBUG "result": [ 20:30:58 DEBUG { 20:30:58 DEBUG "metric": { 20:30:58 DEBUG "pod": "ds-cts-0" 20:30:58 DEBUG }, 20:30:58 DEBUG "value": [ 20:30:58 DEBUG 20:30:58 DEBUG 20:30:58 DEBUG 20:30:58 DEBUG [...] 20:30:58 DEBUG 20:30:58 DEBUG 20:30:58 DEBUG 20:30:58 DEBUG "pod": "ds-cts-2" 20:30:58 DEBUG }, 20:30:58 DEBUG "value": [ 20:30:58 DEBUG 1713641458, 20:30:58 DEBUG "0" 20:30:58 DEBUG ] 20:30:58 DEBUG } 20:30:58 DEBUG ] 20:30:58 DEBUG } 20:30:58 DEBUG } 20:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713641459" 20:30:59 INFO [http_cmd]: http status code OK 20:30:59 DEBUG --- status code --- 20:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:59 DEBUG --- http response --- 20:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:30:59 DEBUG ----- output ----- 20:30:59 DEBUG { 20:30:59 DEBUG "status": "success", 20:30:59 DEBUG "data": { 20:30:59 DEBUG "resultType": "vector", 20:30:59 DEBUG "result": [ 20:30:59 DEBUG { 20:30:59 DEBUG "metric": { 20:30:59 DEBUG "pod": "ds-idrepo-0" 20:30:59 DEBUG }, 20:30:59 DEBUG "value": [ 20:30:59 DEBUG 20:30:59 DEBUG 20:30:59 DEBUG 20:30:59 DEBUG [...] 20:30:59 DEBUG 20:30:59 DEBUG 20:30:59 DEBUG 20:30:59 DEBUG "pod": "ds-cts-2" 20:30:59 DEBUG }, 20:30:59 DEBUG "value": [ 20:30:59 DEBUG 1713641459, 20:30:59 DEBUG "0" 20:30:59 DEBUG ] 20:30:59 DEBUG } 20:30:59 DEBUG ] 20:30:59 DEBUG } 20:30:59 DEBUG } 20:30:59 INFO 20:30:59 INFO [loop_until]: kubectl --namespace=xlou top pods 20:30:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:30:59 INFO [loop_until]: OK (rc = 0) 20:30:59 DEBUG --- stdout --- 20:30:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 7m 8603Mi am-767f6b67b5-qcqz2 9m 9018Mi am-767f6b67b5-vbltz 6m 8609Mi ds-cts-0 23m 11068Mi ds-cts-1 17m 10805Mi ds-cts-2 17m 11055Mi ds-idrepo-0 14m 3895Mi ds-idrepo-1 31m 3878Mi ds-idrepo-2 14m 3837Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 1m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 998m 2673Mi 20:30:59 DEBUG --- stderr --- 20:30:59 DEBUG 20:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641459" 20:30:59 INFO [http_cmd]: http status code OK 20:30:59 DEBUG --- status code --- 20:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:59 DEBUG --- http response --- 20:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:30:59 DEBUG ----- output ----- 20:30:59 DEBUG { 20:30:59 DEBUG "status": "success", 20:30:59 DEBUG "data": { 20:30:59 DEBUG "resultType": "vector", 20:30:59 DEBUG "result": [ 20:30:59 DEBUG { 20:30:59 DEBUG "metric": { 20:30:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:30:59 DEBUG }, 20:30:59 DEBUG "value": [ 20:30:59 DEBUG 20:30:59 DEBUG 20:30:59 DEBUG 20:30:59 DEBUG [...] 20:30:59 DEBUG 20:30:59 DEBUG 20:30:59 DEBUG 20:30:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:30:59 DEBUG }, 20:30:59 DEBUG "value": [ 20:30:59 DEBUG 1713641459, 20:30:59 DEBUG "0" 20:30:59 DEBUG ] 20:30:59 DEBUG } 20:30:59 DEBUG ] 20:30:59 DEBUG } 20:30:59 DEBUG } 20:30:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641459" 20:30:59 INFO [http_cmd]: http status code OK 20:30:59 DEBUG --- status code --- 20:30:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:30:59 DEBUG --- http response --- 20:30:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:30:59 DEBUG ----- output ----- 20:30:59 DEBUG { 20:30:59 DEBUG "status": "success", 20:30:59 DEBUG "data": { 20:30:59 DEBUG "resultType": "vector", 20:30:59 DEBUG "result": [ 20:30:59 DEBUG { 20:30:59 DEBUG "metric": { 20:30:59 DEBUG "instance": "10.142.0.107:9100" 20:30:59 DEBUG }, 20:30:59 DEBUG "value": [ 20:30:59 DEBUG 20:30:59 DEBUG 20:30:59 DEBUG 20:30:59 DEBUG [...] 20:30:59 DEBUG 20:30:59 DEBUG 20:30:59 DEBUG 20:30:59 DEBUG "instance": "10.142.0.43:9100" 20:30:59 DEBUG }, 20:30:59 DEBUG "value": [ 20:30:59 DEBUG 1713641459, 20:30:59 DEBUG "0.001233333333260589" 20:30:59 DEBUG ] 20:30:59 DEBUG } 20:30:59 DEBUG ] 20:30:59 DEBUG } 20:30:59 DEBUG } 20:31:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713641460" 20:31:00 INFO [http_cmd]: http status code OK 20:31:00 DEBUG --- status code --- 20:31:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:00 DEBUG --- http response --- 20:31:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:31:00 DEBUG ----- output ----- 20:31:00 DEBUG { 20:31:00 DEBUG "status": "success", 20:31:00 DEBUG "data": { 20:31:00 DEBUG "resultType": "vector", 20:31:00 DEBUG "result": [ 20:31:00 DEBUG { 20:31:00 DEBUG "metric": { 20:31:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG [...] 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 1713641460, 20:31:00 DEBUG "2547585024" 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG } 20:31:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641460" 20:31:00 INFO [http_cmd]: http status code OK 20:31:00 DEBUG --- status code --- 20:31:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:00 DEBUG --- http response --- 20:31:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:31:00 DEBUG ----- output ----- 20:31:00 DEBUG { 20:31:00 DEBUG "status": "success", 20:31:00 DEBUG "data": { 20:31:00 DEBUG "resultType": "vector", 20:31:00 DEBUG "result": [ 20:31:00 DEBUG { 20:31:00 DEBUG "metric": { 20:31:00 DEBUG "instance": "10.142.0.107:9100" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG [...] 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG "instance": "10.142.0.43:9100" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 1713641460, 20:31:00 DEBUG "29059.3" 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG } 20:31:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641460" 20:31:00 INFO [http_cmd]: http status code OK 20:31:00 DEBUG --- status code --- 20:31:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:00 DEBUG --- http response --- 20:31:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:31:00 DEBUG ----- output ----- 20:31:00 DEBUG { 20:31:00 DEBUG "status": "success", 20:31:00 DEBUG "data": { 20:31:00 DEBUG "resultType": "vector", 20:31:00 DEBUG "result": [ 20:31:00 DEBUG { 20:31:00 DEBUG "metric": { 20:31:00 DEBUG "instance": "10.142.0.107:9100" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG [...] 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG "instance": "10.142.0.43:9100" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 1713641460, 20:31:00 DEBUG "33027.86666666667" 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG } 20:31:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641460" 20:31:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641460" 20:31:00 INFO [http_cmd]: http status code OK 20:31:00 DEBUG --- status code --- 20:31:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:00 DEBUG --- http response --- 20:31:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:31:00 DEBUG ----- output ----- 20:31:00 DEBUG { 20:31:00 DEBUG "status": "success", 20:31:00 DEBUG "data": { 20:31:00 DEBUG "resultType": "vector", 20:31:00 DEBUG "result": [ 20:31:00 DEBUG { 20:31:00 DEBUG "metric": { 20:31:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG [...] 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG "pod": "overseer-0-66cd7d7695-bdwll" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 1713641460, 20:31:00 DEBUG "0" 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG } 20:31:00 INFO [http_cmd]: http status code OK 20:31:00 DEBUG --- status code --- 20:31:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:00 DEBUG --- http response --- 20:31:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:31:00 DEBUG ----- output ----- 20:31:00 DEBUG { 20:31:00 DEBUG "status": "success", 20:31:00 DEBUG "data": { 20:31:00 DEBUG "resultType": "vector", 20:31:00 DEBUG "result": [ 20:31:00 DEBUG { 20:31:00 DEBUG "metric": { 20:31:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG [...] 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 1713641460, 20:31:00 DEBUG "0" 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG } 20:31:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641460" 20:31:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641460" 20:31:00 INFO [http_cmd]: http status code OK 20:31:00 DEBUG --- status code --- 20:31:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:00 DEBUG --- http response --- 20:31:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 20:31:00 DEBUG ----- output ----- 20:31:00 DEBUG { 20:31:00 DEBUG "status": "success", 20:31:00 DEBUG "data": { 20:31:00 DEBUG "resultType": "vector", 20:31:00 DEBUG "result": [ 20:31:00 DEBUG { 20:31:00 DEBUG "metric": { 20:31:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG [...] 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG "pod": "am-767f6b67b5-qcqz2" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 1713641460, 20:31:00 DEBUG "0" 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG } 20:31:00 INFO [http_cmd]: http status code OK 20:31:00 DEBUG --- status code --- 20:31:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:00 DEBUG --- http response --- 20:31:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:31:00 DEBUG ----- output ----- 20:31:00 DEBUG { 20:31:00 DEBUG "status": "success", 20:31:00 DEBUG "data": { 20:31:00 DEBUG "resultType": "vector", 20:31:00 DEBUG "result": [ 20:31:00 DEBUG { 20:31:00 DEBUG "metric": { 20:31:00 DEBUG "pod": "ds-cts-0" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG [...] 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG "pod": "ds-cts-2" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 1713641460, 20:31:00 DEBUG "30" 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG } 20:31:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641460" 20:31:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641460" 20:31:00 INFO [http_cmd]: http status code OK 20:31:00 DEBUG --- status code --- 20:31:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:00 DEBUG --- http response --- 20:31:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:31:00 DEBUG ----- output ----- 20:31:00 DEBUG { 20:31:00 DEBUG "status": "success", 20:31:00 DEBUG "data": { 20:31:00 DEBUG "resultType": "vector", 20:31:00 DEBUG "result": [ 20:31:00 DEBUG { 20:31:00 DEBUG "metric": { 20:31:00 DEBUG "pod": "ds-cts-0" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG [...] 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG "pod": "ds-cts-2" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 1713641460, 20:31:00 DEBUG "0" 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG } 20:31:00 INFO [http_cmd]: http status code OK 20:31:00 DEBUG --- status code --- 20:31:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:00 DEBUG --- http response --- 20:31:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:31:00 DEBUG ----- output ----- 20:31:00 DEBUG { 20:31:00 DEBUG "status": "success", 20:31:00 DEBUG "data": { 20:31:00 DEBUG "resultType": "vector", 20:31:00 DEBUG "result": [ 20:31:00 DEBUG { 20:31:00 DEBUG "metric": { 20:31:00 DEBUG "instance": "10.142.0.107:9100" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG [...] 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG "instance": "10.142.0.43:9100" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 1713641460, 20:31:00 DEBUG "0.0003333333333330302" 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG } 20:31:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713641460" 20:31:00 INFO [http_cmd]: http status code OK 20:31:00 DEBUG --- status code --- 20:31:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:00 DEBUG --- http response --- 20:31:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:31:00 DEBUG ----- output ----- 20:31:00 DEBUG { 20:31:00 DEBUG "status": "success", 20:31:00 DEBUG "data": { 20:31:00 DEBUG "resultType": "vector", 20:31:00 DEBUG "result": [ 20:31:00 DEBUG { 20:31:00 DEBUG "metric": { 20:31:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG [...] 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG 20:31:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:31:00 DEBUG }, 20:31:00 DEBUG "value": [ 20:31:00 DEBUG 1713641460, 20:31:00 DEBUG "0.159" 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG ] 20:31:00 DEBUG } 20:31:00 DEBUG } 20:31:01 INFO 20:31:01 INFO [loop_until]: kubectl --namespace=xlou top node 20:31:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:31:01 INFO [loop_until]: OK (rc = 0) 20:31:01 DEBUG --- stdout --- 20:31:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6209Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 69m 0% 7627Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 56m 0% 11341Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3346Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 77m 0% 14254Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5202Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 61m 0% 11914Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 74m 0% 2934Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 58m 0% 10861Mi 9% gke-xlou-cdm-ds-0f601751-j488 67m 0% 7222Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 74m 0% 15845Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 91m 0% 7847Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 72m 0% 13724Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 1145m 7% 8268Mi 14% 20:31:01 DEBUG --- stderr --- 20:31:01 DEBUG 20:31:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641515" 20:31:55 INFO [http_cmd]: http status code OK 20:31:55 DEBUG --- status code --- 20:31:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:55 DEBUG --- http response --- 20:31:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:31:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713641515" 20:31:55 ERROR [http_cmd]: ERROR 20:31:55 ERROR --- status code --- 20:31:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:31:55 ERROR --- http response --- 20:31:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:31:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:31:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641516" 20:31:56 INFO [http_cmd]: http status code OK 20:31:56 DEBUG --- status code --- 20:31:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:56 DEBUG --- http response --- 20:31:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:31:56 DEBUG ----- output ----- 20:31:56 DEBUG { 20:31:56 DEBUG "status": "success", 20:31:56 DEBUG "data": { 20:31:56 DEBUG "resultType": "vector", 20:31:56 DEBUG "result": [ 20:31:56 DEBUG { 20:31:56 DEBUG "metric": { 20:31:56 DEBUG "pod": "ds-idrepo-0" 20:31:56 DEBUG }, 20:31:56 DEBUG "value": [ 20:31:56 DEBUG 20:31:56 DEBUG 20:31:56 DEBUG 20:31:56 DEBUG [...] 20:31:56 DEBUG 20:31:56 DEBUG 20:31:56 DEBUG 20:31:56 DEBUG "pod": "ds-idrepo-2" 20:31:56 DEBUG }, 20:31:56 DEBUG "value": [ 20:31:56 DEBUG 1713641516, 20:31:56 DEBUG "0" 20:31:56 DEBUG ] 20:31:56 DEBUG } 20:31:56 DEBUG ] 20:31:56 DEBUG } 20:31:56 DEBUG } 20:31:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641516" 20:31:56 INFO [http_cmd]: http status code OK 20:31:56 DEBUG --- status code --- 20:31:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:56 DEBUG --- http response --- 20:31:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:31:56 DEBUG ----- output ----- 20:31:56 DEBUG { 20:31:56 DEBUG "status": "success", 20:31:56 DEBUG "data": { 20:31:56 DEBUG "resultType": "vector", 20:31:56 DEBUG "result": [ 20:31:56 DEBUG { 20:31:56 DEBUG "metric": { 20:31:56 DEBUG "pod": "ds-idrepo-0" 20:31:56 DEBUG }, 20:31:56 DEBUG "value": [ 20:31:56 DEBUG 20:31:56 DEBUG 20:31:56 DEBUG 20:31:56 DEBUG [...] 20:31:56 DEBUG 20:31:56 DEBUG 20:31:56 DEBUG 20:31:56 DEBUG "pod": "ds-idrepo-2" 20:31:56 DEBUG }, 20:31:56 DEBUG "value": [ 20:31:56 DEBUG 1713641516, 20:31:56 DEBUG "0" 20:31:56 DEBUG ] 20:31:56 DEBUG } 20:31:56 DEBUG ] 20:31:56 DEBUG } 20:31:56 DEBUG } 20:31:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641516" 20:31:56 INFO [http_cmd]: http status code OK 20:31:56 DEBUG --- status code --- 20:31:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:56 DEBUG --- http response --- 20:31:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:31:56 DEBUG ----- output ----- 20:31:56 DEBUG { 20:31:56 DEBUG "status": "success", 20:31:56 DEBUG "data": { 20:31:56 DEBUG "resultType": "vector", 20:31:56 DEBUG "result": [ 20:31:56 DEBUG { 20:31:56 DEBUG "metric": { 20:31:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:31:56 DEBUG }, 20:31:56 DEBUG "value": [ 20:31:56 DEBUG 20:31:56 DEBUG 20:31:56 DEBUG 20:31:56 DEBUG [...] 20:31:56 DEBUG 20:31:56 DEBUG 20:31:56 DEBUG 20:31:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:31:56 DEBUG }, 20:31:56 DEBUG "value": [ 20:31:56 DEBUG 1713641516, 20:31:56 DEBUG "0" 20:31:56 DEBUG ] 20:31:56 DEBUG } 20:31:56 DEBUG ] 20:31:56 DEBUG } 20:31:56 DEBUG } 20:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641517" 20:31:57 INFO [http_cmd]: http status code OK 20:31:57 DEBUG --- status code --- 20:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:57 DEBUG --- http response --- 20:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:31:57 DEBUG ----- output ----- 20:31:57 DEBUG { 20:31:57 DEBUG "status": "success", 20:31:57 DEBUG "data": { 20:31:57 DEBUG "resultType": "vector", 20:31:57 DEBUG "result": [ 20:31:57 DEBUG { 20:31:57 DEBUG "metric": { 20:31:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:31:57 DEBUG }, 20:31:57 DEBUG "value": [ 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG [...] 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG "pod": "ds-idrepo-2" 20:31:57 DEBUG }, 20:31:57 DEBUG "value": [ 20:31:57 DEBUG 1713641517, 20:31:57 DEBUG "4537.774283467944" 20:31:57 DEBUG ] 20:31:57 DEBUG } 20:31:57 DEBUG ] 20:31:57 DEBUG } 20:31:57 DEBUG } 20:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641517" 20:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641517" 20:31:57 INFO [http_cmd]: http status code OK 20:31:57 INFO [http_cmd]: http status code OK 20:31:57 DEBUG --- status code --- 20:31:57 DEBUG --- status code --- 20:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:57 DEBUG --- http response --- 20:31:57 DEBUG --- http response --- 20:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:31:57 DEBUG ----- output ----- 20:31:57 DEBUG ----- output ----- 20:31:57 DEBUG { 20:31:57 DEBUG { 20:31:57 DEBUG "status": "success", 20:31:57 DEBUG "status": "success", 20:31:57 DEBUG "data": { 20:31:57 DEBUG "data": { 20:31:57 DEBUG "resultType": "vector", 20:31:57 DEBUG "resultType": "vector", 20:31:57 DEBUG "result": [ 20:31:57 DEBUG "result": [ 20:31:57 DEBUG { 20:31:57 DEBUG { 20:31:57 DEBUG "metric": { 20:31:57 DEBUG "metric": { 20:31:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:31:57 DEBUG "instance": "10.142.0.107:9100" 20:31:57 DEBUG }, 20:31:57 DEBUG }, 20:31:57 DEBUG "value": [ 20:31:57 DEBUG "value": [ 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG [...] 20:31:57 DEBUG [...] 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:31:57 DEBUG "instance": "10.142.0.43:9100" 20:31:57 DEBUG }, 20:31:57 DEBUG }, 20:31:57 DEBUG "value": [ 20:31:57 DEBUG "value": [ 20:31:57 DEBUG 1713641517, 20:31:57 DEBUG 1713641517, 20:31:57 DEBUG "0" 20:31:57 DEBUG "3764103037952" 20:31:57 DEBUG ] 20:31:57 DEBUG ] 20:31:57 DEBUG } 20:31:57 DEBUG } 20:31:57 DEBUG ] 20:31:57 DEBUG ] 20:31:57 DEBUG } 20:31:57 DEBUG } 20:31:57 DEBUG } 20:31:57 DEBUG } 20:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713641517" 20:31:57 INFO [http_cmd]: http status code OK 20:31:57 DEBUG --- status code --- 20:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:57 DEBUG --- http response --- 20:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:31:57 DEBUG ----- output ----- 20:31:57 DEBUG { 20:31:57 DEBUG "status": "success", 20:31:57 DEBUG "data": { 20:31:57 DEBUG "resultType": "vector", 20:31:57 DEBUG "result": [ 20:31:57 DEBUG { 20:31:57 DEBUG "metric": { 20:31:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:31:57 DEBUG }, 20:31:57 DEBUG "value": [ 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG [...] 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:31:57 DEBUG }, 20:31:57 DEBUG "value": [ 20:31:57 DEBUG 1713641517, 20:31:57 DEBUG "1.026182447639953" 20:31:57 DEBUG ] 20:31:57 DEBUG } 20:31:57 DEBUG ] 20:31:57 DEBUG } 20:31:57 DEBUG } 20:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641517" 20:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641517" 20:31:57 INFO [http_cmd]: http status code OK 20:31:57 DEBUG --- status code --- 20:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:57 DEBUG --- http response --- 20:31:57 INFO [http_cmd]: http status code OK 20:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:31:57 DEBUG --- status code --- 20:31:57 DEBUG ----- output ----- 20:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:57 DEBUG { 20:31:57 DEBUG --- http response --- 20:31:57 DEBUG "status": "success", 20:31:57 DEBUG "data": { 20:31:57 DEBUG "resultType": "vector", 20:31:57 DEBUG "result": [ 20:31:57 DEBUG { 20:31:57 DEBUG "metric": { 20:31:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:31:57 DEBUG }, 20:31:57 DEBUG "value": [ 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG [...] 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:31:57 DEBUG }, 20:31:57 DEBUG "value": [ 20:31:57 DEBUG 1713641517, 20:31:57 DEBUG "0" 20:31:57 DEBUG ] 20:31:57 DEBUG } 20:31:57 DEBUG ] 20:31:57 DEBUG } 20:31:57 DEBUG } 20:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:31:57 DEBUG ----- output ----- 20:31:57 DEBUG { 20:31:57 DEBUG "status": "success", 20:31:57 DEBUG "data": { 20:31:57 DEBUG "resultType": "vector", 20:31:57 DEBUG "result": [ 20:31:57 DEBUG { 20:31:57 DEBUG "metric": { 20:31:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:31:57 DEBUG }, 20:31:57 DEBUG "value": [ 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG [...] 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG 20:31:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:31:57 DEBUG }, 20:31:57 DEBUG "value": [ 20:31:57 DEBUG 1713641517, 20:31:57 DEBUG "0" 20:31:57 DEBUG ] 20:31:57 DEBUG } 20:31:57 DEBUG ] 20:31:57 DEBUG } 20:31:57 DEBUG } 20:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641518" 20:31:58 INFO [http_cmd]: http status code OK 20:31:58 DEBUG --- status code --- 20:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:58 DEBUG --- http response --- 20:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:31:58 DEBUG ----- output ----- 20:31:58 DEBUG { 20:31:58 DEBUG "status": "success", 20:31:58 DEBUG "data": { 20:31:58 DEBUG "resultType": "vector", 20:31:58 DEBUG "result": [ 20:31:58 DEBUG { 20:31:58 DEBUG "metric": { 20:31:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:31:58 DEBUG }, 20:31:58 DEBUG "value": [ 20:31:58 DEBUG 20:31:58 DEBUG 20:31:58 DEBUG 20:31:58 DEBUG [...] 20:31:58 DEBUG 20:31:58 DEBUG 20:31:58 DEBUG 20:31:58 DEBUG "pod": "ds-idrepo-2" 20:31:58 DEBUG }, 20:31:58 DEBUG "value": [ 20:31:58 DEBUG 1713641518, 20:31:58 DEBUG "0.012037238222971234" 20:31:58 DEBUG ] 20:31:58 DEBUG } 20:31:58 DEBUG ] 20:31:58 DEBUG } 20:31:58 DEBUG } 20:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641518" 20:31:58 INFO [http_cmd]: http status code OK 20:31:58 DEBUG --- status code --- 20:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:58 DEBUG --- http response --- 20:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:31:58 DEBUG ----- output ----- 20:31:58 DEBUG { 20:31:58 DEBUG "status": "success", 20:31:58 DEBUG "data": { 20:31:58 DEBUG "resultType": "vector", 20:31:58 DEBUG "result": [ 20:31:58 DEBUG { 20:31:58 DEBUG "metric": { 20:31:58 DEBUG "pod": "ds-idrepo-0" 20:31:58 DEBUG }, 20:31:58 DEBUG "value": [ 20:31:58 DEBUG 20:31:58 DEBUG 20:31:58 DEBUG 20:31:58 DEBUG [...] 20:31:58 DEBUG 20:31:58 DEBUG 20:31:58 DEBUG 20:31:58 DEBUG "pod": "ds-idrepo-2" 20:31:58 DEBUG }, 20:31:58 DEBUG "value": [ 20:31:58 DEBUG 1713641518, 20:31:58 DEBUG "0" 20:31:58 DEBUG ] 20:31:58 DEBUG } 20:31:58 DEBUG ] 20:31:58 DEBUG } 20:31:58 DEBUG } 20:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641519" 20:31:59 INFO [http_cmd]: http status code OK 20:31:59 DEBUG --- status code --- 20:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:59 DEBUG --- http response --- 20:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:31:59 DEBUG ----- output ----- 20:31:59 DEBUG { 20:31:59 DEBUG "status": "success", 20:31:59 DEBUG "data": { 20:31:59 DEBUG "resultType": "vector", 20:31:59 DEBUG "result": [ 20:31:59 DEBUG { 20:31:59 DEBUG "metric": { 20:31:59 DEBUG "pod": "ds-cts-0" 20:31:59 DEBUG }, 20:31:59 DEBUG "value": [ 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG [...] 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG "pod": "ds-cts-2" 20:31:59 DEBUG }, 20:31:59 DEBUG "value": [ 20:31:59 DEBUG 1713641519, 20:31:59 DEBUG "0" 20:31:59 DEBUG ] 20:31:59 DEBUG } 20:31:59 DEBUG ] 20:31:59 DEBUG } 20:31:59 DEBUG } 20:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713641519" 20:31:59 INFO [http_cmd]: http status code OK 20:31:59 DEBUG --- status code --- 20:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:59 DEBUG --- http response --- 20:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:31:59 DEBUG ----- output ----- 20:31:59 DEBUG { 20:31:59 DEBUG "status": "success", 20:31:59 DEBUG "data": { 20:31:59 DEBUG "resultType": "vector", 20:31:59 DEBUG "result": [ 20:31:59 DEBUG { 20:31:59 DEBUG "metric": { 20:31:59 DEBUG "pod": "ds-idrepo-0" 20:31:59 DEBUG }, 20:31:59 DEBUG "value": [ 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG [...] 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG "pod": "ds-cts-2" 20:31:59 DEBUG }, 20:31:59 DEBUG "value": [ 20:31:59 DEBUG 1713641519, 20:31:59 DEBUG "0" 20:31:59 DEBUG ] 20:31:59 DEBUG } 20:31:59 DEBUG ] 20:31:59 DEBUG } 20:31:59 DEBUG } 20:31:59 INFO 20:31:59 INFO [loop_until]: kubectl --namespace=xlou top pods 20:31:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:31:59 INFO [loop_until]: OK (rc = 0) 20:31:59 DEBUG --- stdout --- 20:31:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 7m 8602Mi am-767f6b67b5-qcqz2 7m 9017Mi am-767f6b67b5-vbltz 7m 8610Mi ds-cts-0 19m 11066Mi ds-cts-1 23m 10804Mi ds-cts-2 15m 11055Mi ds-idrepo-0 15m 3895Mi ds-idrepo-1 12m 3878Mi ds-idrepo-2 13m 3837Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 1033m 3443Mi 20:31:59 DEBUG --- stderr --- 20:31:59 DEBUG 20:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641519" 20:31:59 INFO [http_cmd]: http status code OK 20:31:59 DEBUG --- status code --- 20:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:59 DEBUG --- http response --- 20:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:31:59 DEBUG ----- output ----- 20:31:59 DEBUG { 20:31:59 DEBUG "status": "success", 20:31:59 DEBUG "data": { 20:31:59 DEBUG "resultType": "vector", 20:31:59 DEBUG "result": [ 20:31:59 DEBUG { 20:31:59 DEBUG "metric": { 20:31:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:31:59 DEBUG }, 20:31:59 DEBUG "value": [ 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG [...] 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:31:59 DEBUG }, 20:31:59 DEBUG "value": [ 20:31:59 DEBUG 1713641519, 20:31:59 DEBUG "0" 20:31:59 DEBUG ] 20:31:59 DEBUG } 20:31:59 DEBUG ] 20:31:59 DEBUG } 20:31:59 DEBUG } 20:31:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641519" 20:31:59 INFO [http_cmd]: http status code OK 20:31:59 DEBUG --- status code --- 20:31:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:31:59 DEBUG --- http response --- 20:31:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:31:59 DEBUG ----- output ----- 20:31:59 DEBUG { 20:31:59 DEBUG "status": "success", 20:31:59 DEBUG "data": { 20:31:59 DEBUG "resultType": "vector", 20:31:59 DEBUG "result": [ 20:31:59 DEBUG { 20:31:59 DEBUG "metric": { 20:31:59 DEBUG "instance": "10.142.0.107:9100" 20:31:59 DEBUG }, 20:31:59 DEBUG "value": [ 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG [...] 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG 20:31:59 DEBUG "instance": "10.142.0.43:9100" 20:31:59 DEBUG }, 20:31:59 DEBUG "value": [ 20:31:59 DEBUG 1713641519, 20:31:59 DEBUG "0.0010666666666679704" 20:31:59 DEBUG ] 20:31:59 DEBUG } 20:31:59 DEBUG ] 20:31:59 DEBUG } 20:31:59 DEBUG } 20:32:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713641520" 20:32:00 INFO [http_cmd]: http status code OK 20:32:00 DEBUG --- status code --- 20:32:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:00 DEBUG --- http response --- 20:32:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:32:00 DEBUG ----- output ----- 20:32:00 DEBUG { 20:32:00 DEBUG "status": "success", 20:32:00 DEBUG "data": { 20:32:00 DEBUG "resultType": "vector", 20:32:00 DEBUG "result": [ 20:32:00 DEBUG { 20:32:00 DEBUG "metric": { 20:32:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG [...] 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 1713641520, 20:32:00 DEBUG "3293396992" 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG } 20:32:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641520" 20:32:00 INFO [http_cmd]: http status code OK 20:32:00 DEBUG --- status code --- 20:32:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:00 DEBUG --- http response --- 20:32:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:32:00 DEBUG ----- output ----- 20:32:00 DEBUG { 20:32:00 DEBUG "status": "success", 20:32:00 DEBUG "data": { 20:32:00 DEBUG "resultType": "vector", 20:32:00 DEBUG "result": [ 20:32:00 DEBUG { 20:32:00 DEBUG "metric": { 20:32:00 DEBUG "instance": "10.142.0.107:9100" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG [...] 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG "instance": "10.142.0.43:9100" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 1713641520, 20:32:00 DEBUG "26391.266666666666" 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG } 20:32:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641520" 20:32:00 INFO [http_cmd]: http status code OK 20:32:00 DEBUG --- status code --- 20:32:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:00 DEBUG --- http response --- 20:32:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:32:00 DEBUG ----- output ----- 20:32:00 DEBUG { 20:32:00 DEBUG "status": "success", 20:32:00 DEBUG "data": { 20:32:00 DEBUG "resultType": "vector", 20:32:00 DEBUG "result": [ 20:32:00 DEBUG { 20:32:00 DEBUG "metric": { 20:32:00 DEBUG "instance": "10.142.0.107:9100" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG [...] 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG "instance": "10.142.0.43:9100" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 1713641520, 20:32:00 DEBUG "32028.6" 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG } 20:32:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641520" 20:32:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641520" 20:32:00 INFO [http_cmd]: http status code OK 20:32:00 DEBUG --- status code --- 20:32:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:00 DEBUG --- http response --- 20:32:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:32:00 DEBUG ----- output ----- 20:32:00 DEBUG { 20:32:00 DEBUG "status": "success", 20:32:00 DEBUG "data": { 20:32:00 DEBUG "resultType": "vector", 20:32:00 DEBUG "result": [ 20:32:00 DEBUG { 20:32:00 DEBUG "metric": { 20:32:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG [...] 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 1713641520, 20:32:00 DEBUG "0" 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG } 20:32:00 INFO [http_cmd]: http status code OK 20:32:00 DEBUG --- status code --- 20:32:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:00 DEBUG --- http response --- 20:32:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:32:00 DEBUG ----- output ----- 20:32:00 DEBUG { 20:32:00 DEBUG "status": "success", 20:32:00 DEBUG "data": { 20:32:00 DEBUG "resultType": "vector", 20:32:00 DEBUG "result": [ 20:32:00 DEBUG { 20:32:00 DEBUG "metric": { 20:32:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG [...] 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 1713641520, 20:32:00 DEBUG "3988.0566013241596" 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG } 20:32:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641520" 20:32:00 INFO [http_cmd]: http status code OK 20:32:00 DEBUG --- status code --- 20:32:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:00 DEBUG --- http response --- 20:32:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:32:00 DEBUG ----- output ----- 20:32:00 DEBUG { 20:32:00 DEBUG "status": "success", 20:32:00 DEBUG "data": { 20:32:00 DEBUG "resultType": "vector", 20:32:00 DEBUG "result": [ 20:32:00 DEBUG { 20:32:00 DEBUG "metric": { 20:32:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG [...] 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 1713641520, 20:32:00 DEBUG "0" 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG } 20:32:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641520" 20:32:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713641520" 20:32:00 INFO [http_cmd]: http status code OK 20:32:00 DEBUG --- status code --- 20:32:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:00 DEBUG --- http response --- 20:32:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:32:00 DEBUG ----- output ----- 20:32:00 DEBUG { 20:32:00 DEBUG "status": "success", 20:32:00 DEBUG "data": { 20:32:00 DEBUG "resultType": "vector", 20:32:00 DEBUG "result": [ 20:32:00 DEBUG { 20:32:00 DEBUG "metric": { 20:32:00 DEBUG "pod": "ds-cts-0" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG [...] 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG "pod": "ds-cts-2" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 1713641520, 20:32:00 DEBUG "30" 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG } 20:32:00 INFO [http_cmd]: http status code OK 20:32:00 DEBUG --- status code --- 20:32:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:00 DEBUG --- http response --- 20:32:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:32:00 DEBUG ----- output ----- 20:32:00 DEBUG { 20:32:00 DEBUG "status": "success", 20:32:00 DEBUG "data": { 20:32:00 DEBUG "resultType": "vector", 20:32:00 DEBUG "result": [ 20:32:00 DEBUG { 20:32:00 DEBUG "metric": { 20:32:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG [...] 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 1713641520, 20:32:00 DEBUG "0.159" 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG } 20:32:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641520" 20:32:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641520" 20:32:00 INFO [http_cmd]: http status code OK 20:32:00 DEBUG --- status code --- 20:32:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:00 DEBUG --- http response --- 20:32:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:32:00 DEBUG ----- output ----- 20:32:00 DEBUG { 20:32:00 DEBUG "status": "success", 20:32:00 DEBUG "data": { 20:32:00 DEBUG "resultType": "vector", 20:32:00 DEBUG "result": [ 20:32:00 DEBUG { 20:32:00 DEBUG "metric": { 20:32:00 DEBUG "pod": "ds-cts-0" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG [...] 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG "pod": "ds-cts-2" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 1713641520, 20:32:00 DEBUG "0" 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG } 20:32:00 INFO [http_cmd]: http status code OK 20:32:00 DEBUG --- status code --- 20:32:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:00 DEBUG --- http response --- 20:32:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:32:00 DEBUG ----- output ----- 20:32:00 DEBUG { 20:32:00 DEBUG "status": "success", 20:32:00 DEBUG "data": { 20:32:00 DEBUG "resultType": "vector", 20:32:00 DEBUG "result": [ 20:32:00 DEBUG { 20:32:00 DEBUG "metric": { 20:32:00 DEBUG "instance": "10.142.0.107:9100" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG [...] 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG 20:32:00 DEBUG "instance": "10.142.0.43:9100" 20:32:00 DEBUG }, 20:32:00 DEBUG "value": [ 20:32:00 DEBUG 1713641520, 20:32:00 DEBUG "0.0003333333333330302" 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG ] 20:32:00 DEBUG } 20:32:00 DEBUG } 20:32:01 INFO 20:32:01 INFO [loop_until]: kubectl --namespace=xlou top node 20:32:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:32:01 INFO [loop_until]: OK (rc = 0) 20:32:01 DEBUG --- stdout --- 20:32:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6214Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 70m 0% 7628Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 55m 0% 11327Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3354Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 73m 0% 14246Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5213Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 11899Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 77m 0% 2929Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 60m 0% 10851Mi 9% gke-xlou-cdm-ds-0f601751-j488 68m 0% 7220Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 80m 0% 15838Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 97m 0% 7848Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 68m 0% 13727Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 1079m 6% 9129Mi 15% 20:32:01 DEBUG --- stderr --- 20:32:01 DEBUG 20:32:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641575" 20:32:55 INFO [http_cmd]: http status code OK 20:32:55 DEBUG --- status code --- 20:32:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:55 DEBUG --- http response --- 20:32:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 20:32:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713641575" 20:32:55 ERROR [http_cmd]: ERROR 20:32:55 ERROR --- status code --- 20:32:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 20:32:55 ERROR --- http response --- 20:32:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 20:32:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 20:32:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641576" 20:32:56 INFO [http_cmd]: http status code OK 20:32:56 DEBUG --- status code --- 20:32:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:56 DEBUG --- http response --- 20:32:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:32:56 DEBUG ----- output ----- 20:32:56 DEBUG { 20:32:56 DEBUG "status": "success", 20:32:56 DEBUG "data": { 20:32:56 DEBUG "resultType": "vector", 20:32:56 DEBUG "result": [ 20:32:56 DEBUG { 20:32:56 DEBUG "metric": { 20:32:56 DEBUG "pod": "ds-idrepo-0" 20:32:56 DEBUG }, 20:32:56 DEBUG "value": [ 20:32:56 DEBUG 20:32:56 DEBUG 20:32:56 DEBUG 20:32:56 DEBUG [...] 20:32:56 DEBUG 20:32:56 DEBUG 20:32:56 DEBUG 20:32:56 DEBUG "pod": "ds-idrepo-2" 20:32:56 DEBUG }, 20:32:56 DEBUG "value": [ 20:32:56 DEBUG 1713641576, 20:32:56 DEBUG "0" 20:32:56 DEBUG ] 20:32:56 DEBUG } 20:32:56 DEBUG ] 20:32:56 DEBUG } 20:32:56 DEBUG } 20:32:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641576" 20:32:56 INFO [http_cmd]: http status code OK 20:32:56 DEBUG --- status code --- 20:32:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:56 DEBUG --- http response --- 20:32:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:32:56 DEBUG ----- output ----- 20:32:56 DEBUG { 20:32:56 DEBUG "status": "success", 20:32:56 DEBUG "data": { 20:32:56 DEBUG "resultType": "vector", 20:32:56 DEBUG "result": [ 20:32:56 DEBUG { 20:32:56 DEBUG "metric": { 20:32:56 DEBUG "pod": "ds-idrepo-0" 20:32:56 DEBUG }, 20:32:56 DEBUG "value": [ 20:32:56 DEBUG 20:32:56 DEBUG 20:32:56 DEBUG 20:32:56 DEBUG [...] 20:32:56 DEBUG 20:32:56 DEBUG 20:32:56 DEBUG 20:32:56 DEBUG "pod": "ds-idrepo-2" 20:32:56 DEBUG }, 20:32:56 DEBUG "value": [ 20:32:56 DEBUG 1713641576, 20:32:56 DEBUG "0" 20:32:56 DEBUG ] 20:32:56 DEBUG } 20:32:56 DEBUG ] 20:32:56 DEBUG } 20:32:56 DEBUG } 20:32:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641576" 20:32:56 INFO [http_cmd]: http status code OK 20:32:56 DEBUG --- status code --- 20:32:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:56 DEBUG --- http response --- 20:32:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:32:56 DEBUG ----- output ----- 20:32:56 DEBUG { 20:32:56 DEBUG "status": "success", 20:32:56 DEBUG "data": { 20:32:56 DEBUG "resultType": "vector", 20:32:56 DEBUG "result": [ 20:32:56 DEBUG { 20:32:56 DEBUG "metric": { 20:32:56 DEBUG "pod": "am-767f6b67b5-9zw22" 20:32:56 DEBUG }, 20:32:56 DEBUG "value": [ 20:32:56 DEBUG 20:32:56 DEBUG 20:32:56 DEBUG 20:32:56 DEBUG [...] 20:32:56 DEBUG 20:32:56 DEBUG 20:32:56 DEBUG 20:32:56 DEBUG "pod": "am-767f6b67b5-vbltz" 20:32:56 DEBUG }, 20:32:56 DEBUG "value": [ 20:32:56 DEBUG 1713641576, 20:32:56 DEBUG "0" 20:32:56 DEBUG ] 20:32:56 DEBUG } 20:32:56 DEBUG ] 20:32:56 DEBUG } 20:32:56 DEBUG } 20:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641577" 20:32:57 INFO [http_cmd]: http status code OK 20:32:57 DEBUG --- status code --- 20:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:57 DEBUG --- http response --- 20:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:32:57 DEBUG ----- output ----- 20:32:57 DEBUG { 20:32:57 DEBUG "status": "success", 20:32:57 DEBUG "data": { 20:32:57 DEBUG "resultType": "vector", 20:32:57 DEBUG "result": [ 20:32:57 DEBUG { 20:32:57 DEBUG "metric": { 20:32:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:32:57 DEBUG }, 20:32:57 DEBUG "value": [ 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG [...] 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:32:57 DEBUG }, 20:32:57 DEBUG "value": [ 20:32:57 DEBUG 1713641577, 20:32:57 DEBUG "0" 20:32:57 DEBUG ] 20:32:57 DEBUG } 20:32:57 DEBUG ] 20:32:57 DEBUG } 20:32:57 DEBUG } 20:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641577" 20:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641577" 20:32:57 INFO [http_cmd]: http status code OK 20:32:57 INFO [http_cmd]: http status code OK 20:32:57 DEBUG --- status code --- 20:32:57 DEBUG --- status code --- 20:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:57 DEBUG --- http response --- 20:32:57 DEBUG --- http response --- 20:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:32:57 DEBUG ----- output ----- 20:32:57 DEBUG ----- output ----- 20:32:57 DEBUG { 20:32:57 DEBUG { 20:32:57 DEBUG "status": "success", 20:32:57 DEBUG "status": "success", 20:32:57 DEBUG "data": { 20:32:57 DEBUG "data": { 20:32:57 DEBUG "resultType": "vector", 20:32:57 DEBUG "resultType": "vector", 20:32:57 DEBUG "result": [ 20:32:57 DEBUG "result": [ 20:32:57 DEBUG { 20:32:57 DEBUG { 20:32:57 DEBUG "metric": { 20:32:57 DEBUG "metric": { 20:32:57 DEBUG "pod": "am-767f6b67b5-9zw22" 20:32:57 DEBUG "instance": "10.142.0.107:9100" 20:32:57 DEBUG }, 20:32:57 DEBUG }, 20:32:57 DEBUG "value": [ 20:32:57 DEBUG "value": [ 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG [...] 20:32:57 DEBUG [...] 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG "pod": "am-767f6b67b5-vbltz" 20:32:57 DEBUG "instance": "10.142.0.43:9100" 20:32:57 DEBUG }, 20:32:57 DEBUG }, 20:32:57 DEBUG "value": [ 20:32:57 DEBUG "value": [ 20:32:57 DEBUG 1713641577, 20:32:57 DEBUG 1713641577, 20:32:57 DEBUG "0" 20:32:57 DEBUG "3764104975360" 20:32:57 DEBUG ] 20:32:57 DEBUG ] 20:32:57 DEBUG } 20:32:57 DEBUG } 20:32:57 DEBUG ] 20:32:57 DEBUG ] 20:32:57 DEBUG } 20:32:57 DEBUG } 20:32:57 DEBUG } 20:32:57 DEBUG } 20:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713641577" 20:32:57 INFO [http_cmd]: http status code OK 20:32:57 DEBUG --- status code --- 20:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:57 DEBUG --- http response --- 20:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:32:57 DEBUG ----- output ----- 20:32:57 DEBUG { 20:32:57 DEBUG "status": "success", 20:32:57 DEBUG "data": { 20:32:57 DEBUG "resultType": "vector", 20:32:57 DEBUG "result": [ 20:32:57 DEBUG { 20:32:57 DEBUG "metric": { 20:32:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:32:57 DEBUG }, 20:32:57 DEBUG "value": [ 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG [...] 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG 20:32:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:32:57 DEBUG }, 20:32:57 DEBUG "value": [ 20:32:57 DEBUG 1713641577, 20:32:57 DEBUG "1.0322970081736134" 20:32:57 DEBUG ] 20:32:57 DEBUG } 20:32:57 DEBUG ] 20:32:57 DEBUG } 20:32:57 DEBUG } 20:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641577" 20:32:58 INFO [http_cmd]: http status code OK 20:32:58 DEBUG --- status code --- 20:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:58 DEBUG --- http response --- 20:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:32:58 DEBUG ----- output ----- 20:32:58 DEBUG { 20:32:58 DEBUG "status": "success", 20:32:58 DEBUG "data": { 20:32:58 DEBUG "resultType": "vector", 20:32:58 DEBUG "result": [ 20:32:58 DEBUG { 20:32:58 DEBUG "metric": { 20:32:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:32:58 DEBUG }, 20:32:58 DEBUG "value": [ 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG [...] 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:32:58 DEBUG }, 20:32:58 DEBUG "value": [ 20:32:58 DEBUG 1713641577, 20:32:58 DEBUG "0" 20:32:58 DEBUG ] 20:32:58 DEBUG } 20:32:58 DEBUG ] 20:32:58 DEBUG } 20:32:58 DEBUG } 20:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641578" 20:32:58 INFO [http_cmd]: http status code OK 20:32:58 DEBUG --- status code --- 20:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:58 DEBUG --- http response --- 20:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:32:58 DEBUG ----- output ----- 20:32:58 DEBUG { 20:32:58 DEBUG "status": "success", 20:32:58 DEBUG "data": { 20:32:58 DEBUG "resultType": "vector", 20:32:58 DEBUG "result": [ 20:32:58 DEBUG { 20:32:58 DEBUG "metric": { 20:32:58 DEBUG "pod": "am-767f6b67b5-9zw22" 20:32:58 DEBUG }, 20:32:58 DEBUG "value": [ 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG [...] 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG "pod": "am-767f6b67b5-vbltz" 20:32:58 DEBUG }, 20:32:58 DEBUG "value": [ 20:32:58 DEBUG 1713641578, 20:32:58 DEBUG "0" 20:32:58 DEBUG ] 20:32:58 DEBUG } 20:32:58 DEBUG ] 20:32:58 DEBUG } 20:32:58 DEBUG } 20:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641578" 20:32:58 INFO [http_cmd]: http status code OK 20:32:58 DEBUG --- status code --- 20:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:58 DEBUG --- http response --- 20:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 20:32:58 DEBUG ----- output ----- 20:32:58 DEBUG { 20:32:58 DEBUG "status": "success", 20:32:58 DEBUG "data": { 20:32:58 DEBUG "resultType": "vector", 20:32:58 DEBUG "result": [ 20:32:58 DEBUG { 20:32:58 DEBUG "metric": { 20:32:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:32:58 DEBUG }, 20:32:58 DEBUG "value": [ 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG [...] 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:32:58 DEBUG }, 20:32:58 DEBUG "value": [ 20:32:58 DEBUG 1713641578, 20:32:58 DEBUG "0.000023441424345598422" 20:32:58 DEBUG ] 20:32:58 DEBUG } 20:32:58 DEBUG ] 20:32:58 DEBUG } 20:32:58 DEBUG } 20:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641578" 20:32:58 INFO [http_cmd]: http status code OK 20:32:58 DEBUG --- status code --- 20:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:58 DEBUG --- http response --- 20:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:32:58 DEBUG ----- output ----- 20:32:58 DEBUG { 20:32:58 DEBUG "status": "success", 20:32:58 DEBUG "data": { 20:32:58 DEBUG "resultType": "vector", 20:32:58 DEBUG "result": [ 20:32:58 DEBUG { 20:32:58 DEBUG "metric": { 20:32:58 DEBUG "pod": "ds-idrepo-0" 20:32:58 DEBUG }, 20:32:58 DEBUG "value": [ 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG [...] 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG 20:32:58 DEBUG "pod": "ds-idrepo-2" 20:32:58 DEBUG }, 20:32:58 DEBUG "value": [ 20:32:58 DEBUG 1713641578, 20:32:58 DEBUG "0" 20:32:58 DEBUG ] 20:32:58 DEBUG } 20:32:58 DEBUG ] 20:32:58 DEBUG } 20:32:58 DEBUG } 20:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641579" 20:32:59 INFO [http_cmd]: http status code OK 20:32:59 DEBUG --- status code --- 20:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:59 DEBUG --- http response --- 20:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:32:59 DEBUG ----- output ----- 20:32:59 DEBUG { 20:32:59 DEBUG "status": "success", 20:32:59 DEBUG "data": { 20:32:59 DEBUG "resultType": "vector", 20:32:59 DEBUG "result": [ 20:32:59 DEBUG { 20:32:59 DEBUG "metric": { 20:32:59 DEBUG "pod": "ds-cts-0" 20:32:59 DEBUG }, 20:32:59 DEBUG "value": [ 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG [...] 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG "pod": "ds-cts-2" 20:32:59 DEBUG }, 20:32:59 DEBUG "value": [ 20:32:59 DEBUG 1713641579, 20:32:59 DEBUG "0" 20:32:59 DEBUG ] 20:32:59 DEBUG } 20:32:59 DEBUG ] 20:32:59 DEBUG } 20:32:59 DEBUG } 20:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713641579" 20:32:59 INFO [http_cmd]: http status code OK 20:32:59 DEBUG --- status code --- 20:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:59 DEBUG --- http response --- 20:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:32:59 DEBUG ----- output ----- 20:32:59 DEBUG { 20:32:59 DEBUG "status": "success", 20:32:59 DEBUG "data": { 20:32:59 DEBUG "resultType": "vector", 20:32:59 DEBUG "result": [ 20:32:59 DEBUG { 20:32:59 DEBUG "metric": { 20:32:59 DEBUG "pod": "ds-idrepo-0" 20:32:59 DEBUG }, 20:32:59 DEBUG "value": [ 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG [...] 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG "pod": "ds-cts-2" 20:32:59 DEBUG }, 20:32:59 DEBUG "value": [ 20:32:59 DEBUG 1713641579, 20:32:59 DEBUG "0" 20:32:59 DEBUG ] 20:32:59 DEBUG } 20:32:59 DEBUG ] 20:32:59 DEBUG } 20:32:59 DEBUG } 20:32:59 INFO 20:32:59 INFO [loop_until]: kubectl --namespace=xlou top pods 20:32:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641579" 20:32:59 INFO [http_cmd]: http status code OK 20:32:59 DEBUG --- status code --- 20:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:59 DEBUG --- http response --- 20:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:32:59 DEBUG ----- output ----- 20:32:59 DEBUG { 20:32:59 DEBUG "status": "success", 20:32:59 DEBUG "data": { 20:32:59 DEBUG "resultType": "vector", 20:32:59 DEBUG "result": [ 20:32:59 DEBUG { 20:32:59 DEBUG "metric": { 20:32:59 DEBUG "pod": "am-767f6b67b5-9zw22" 20:32:59 DEBUG }, 20:32:59 DEBUG "value": [ 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG [...] 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG "pod": "am-767f6b67b5-vbltz" 20:32:59 DEBUG }, 20:32:59 DEBUG "value": [ 20:32:59 DEBUG 1713641579, 20:32:59 DEBUG "0" 20:32:59 DEBUG ] 20:32:59 DEBUG } 20:32:59 DEBUG ] 20:32:59 DEBUG } 20:32:59 DEBUG } 20:32:59 INFO [loop_until]: OK (rc = 0) 20:32:59 DEBUG --- stdout --- 20:32:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 7m 8602Mi am-767f6b67b5-qcqz2 7m 9017Mi am-767f6b67b5-vbltz 6m 8609Mi ds-cts-0 18m 11067Mi ds-cts-1 17m 10805Mi ds-cts-2 15m 11056Mi ds-idrepo-0 14m 3895Mi ds-idrepo-1 12m 3878Mi ds-idrepo-2 13m 3836Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1178Mi idm-b85f9d978-wnhxb 3m 1238Mi lodemon-6f4955bbbb-clfwf 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-66cd7d7695-bdwll 762m 3849Mi 20:32:59 DEBUG --- stderr --- 20:32:59 DEBUG 20:32:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641579" 20:32:59 INFO [http_cmd]: http status code OK 20:32:59 DEBUG --- status code --- 20:32:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:32:59 DEBUG --- http response --- 20:32:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:32:59 DEBUG ----- output ----- 20:32:59 DEBUG { 20:32:59 DEBUG "status": "success", 20:32:59 DEBUG "data": { 20:32:59 DEBUG "resultType": "vector", 20:32:59 DEBUG "result": [ 20:32:59 DEBUG { 20:32:59 DEBUG "metric": { 20:32:59 DEBUG "instance": "10.142.0.107:9100" 20:32:59 DEBUG }, 20:32:59 DEBUG "value": [ 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG [...] 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG 20:32:59 DEBUG "instance": "10.142.0.43:9100" 20:32:59 DEBUG }, 20:32:59 DEBUG "value": [ 20:32:59 DEBUG 1713641579, 20:32:59 DEBUG "0.0010333333333268986" 20:32:59 DEBUG ] 20:32:59 DEBUG } 20:32:59 DEBUG ] 20:32:59 DEBUG } 20:32:59 DEBUG } 20:33:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713641580" 20:33:00 INFO [http_cmd]: http status code OK 20:33:00 DEBUG --- status code --- 20:33:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:33:00 DEBUG --- http response --- 20:33:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:33:00 DEBUG ----- output ----- 20:33:00 DEBUG { 20:33:00 DEBUG "status": "success", 20:33:00 DEBUG "data": { 20:33:00 DEBUG "resultType": "vector", 20:33:00 DEBUG "result": [ 20:33:00 DEBUG { 20:33:00 DEBUG "metric": { 20:33:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG [...] 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 1713641580, 20:33:00 DEBUG "4182118400" 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG } 20:33:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641580" 20:33:00 INFO [http_cmd]: http status code OK 20:33:00 DEBUG --- status code --- 20:33:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:33:00 DEBUG --- http response --- 20:33:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:33:00 DEBUG ----- output ----- 20:33:00 DEBUG { 20:33:00 DEBUG "status": "success", 20:33:00 DEBUG "data": { 20:33:00 DEBUG "resultType": "vector", 20:33:00 DEBUG "result": [ 20:33:00 DEBUG { 20:33:00 DEBUG "metric": { 20:33:00 DEBUG "instance": "10.142.0.107:9100" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG [...] 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG "instance": "10.142.0.43:9100" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 1713641580, 20:33:00 DEBUG "27009.8" 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG } 20:33:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641580" 20:33:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641580" 20:33:00 INFO [http_cmd]: http status code OK 20:33:00 DEBUG --- status code --- 20:33:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:33:00 DEBUG --- http response --- 20:33:00 INFO [http_cmd]: http status code OK 20:33:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:33:00 DEBUG --- status code --- 20:33:00 DEBUG ----- output ----- 20:33:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:33:00 DEBUG { 20:33:00 DEBUG --- http response --- 20:33:00 DEBUG "status": "success", 20:33:00 DEBUG "data": { 20:33:00 DEBUG "resultType": "vector", 20:33:00 DEBUG "result": [ 20:33:00 DEBUG { 20:33:00 DEBUG "metric": { 20:33:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:33:00 DEBUG 20:33:00 DEBUG ----- output ----- 20:33:00 DEBUG 20:33:00 DEBUG { 20:33:00 DEBUG 20:33:00 DEBUG "status": "success", 20:33:00 DEBUG [...] 20:33:00 DEBUG "data": { 20:33:00 DEBUG 20:33:00 DEBUG "resultType": "vector", 20:33:00 DEBUG 20:33:00 DEBUG "result": [ 20:33:00 DEBUG 20:33:00 DEBUG { 20:33:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:33:00 DEBUG "metric": { 20:33:00 DEBUG }, 20:33:00 DEBUG "instance": "10.142.0.107:9100" 20:33:00 DEBUG "value": [ 20:33:00 DEBUG }, 20:33:00 DEBUG 1713641580, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG "0" 20:33:00 DEBUG 20:33:00 DEBUG ] 20:33:00 DEBUG 20:33:00 DEBUG } 20:33:00 DEBUG 20:33:00 DEBUG ] 20:33:00 DEBUG [...] 20:33:00 DEBUG } 20:33:00 DEBUG 20:33:00 DEBUG } 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG "instance": "10.142.0.43:9100" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 1713641580, 20:33:00 DEBUG "32426.2" 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG } 20:33:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641580" 20:33:00 INFO [http_cmd]: http status code OK 20:33:00 DEBUG --- status code --- 20:33:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:33:00 DEBUG --- http response --- 20:33:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 20:33:00 DEBUG ----- output ----- 20:33:00 DEBUG { 20:33:00 DEBUG "status": "success", 20:33:00 DEBUG "data": { 20:33:00 DEBUG "resultType": "vector", 20:33:00 DEBUG "result": [ 20:33:00 DEBUG { 20:33:00 DEBUG "metric": { 20:33:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG [...] 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 1713641580, 20:33:00 DEBUG "0" 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG } 20:33:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641580" 20:33:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713641580" 20:33:00 INFO [http_cmd]: http status code OK 20:33:00 DEBUG --- status code --- 20:33:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:33:00 DEBUG --- http response --- 20:33:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:33:00 DEBUG ----- output ----- 20:33:00 DEBUG { 20:33:00 DEBUG "status": "success", 20:33:00 DEBUG "data": { 20:33:00 DEBUG "resultType": "vector", 20:33:00 DEBUG "result": [ 20:33:00 DEBUG { 20:33:00 DEBUG "metric": { 20:33:00 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG [...] 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 1713641580, 20:33:00 DEBUG "0" 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG } 20:33:00 INFO [http_cmd]: http status code OK 20:33:00 DEBUG --- status code --- 20:33:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:33:00 DEBUG --- http response --- 20:33:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 20:33:00 DEBUG ----- output ----- 20:33:00 DEBUG { 20:33:00 DEBUG "status": "success", 20:33:00 DEBUG "data": { 20:33:00 DEBUG "resultType": "vector", 20:33:00 DEBUG "result": [ 20:33:00 DEBUG { 20:33:00 DEBUG "metric": { 20:33:00 DEBUG "pod": "am-767f6b67b5-9zw22" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG [...] 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG "pod": "am-767f6b67b5-vbltz" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 1713641580, 20:33:00 DEBUG "0.159" 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG } 20:33:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641580" 20:33:00 INFO [http_cmd]: http status code OK 20:33:00 DEBUG --- status code --- 20:33:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:33:00 DEBUG --- http response --- 20:33:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:33:00 DEBUG ----- output ----- 20:33:00 DEBUG { 20:33:00 DEBUG "status": "success", 20:33:00 DEBUG "data": { 20:33:00 DEBUG "resultType": "vector", 20:33:00 DEBUG "result": [ 20:33:00 DEBUG { 20:33:00 DEBUG "metric": { 20:33:00 DEBUG "pod": "ds-cts-0" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG [...] 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG "pod": "ds-cts-2" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 1713641580, 20:33:00 DEBUG "30" 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG } 20:33:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713641580" 20:33:00 INFO [http_cmd]: http status code OK 20:33:00 DEBUG --- status code --- 20:33:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:33:00 DEBUG --- http response --- 20:33:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 20:33:00 DEBUG ----- output ----- 20:33:00 DEBUG { 20:33:00 DEBUG "status": "success", 20:33:00 DEBUG "data": { 20:33:00 DEBUG "resultType": "vector", 20:33:00 DEBUG "result": [ 20:33:00 DEBUG { 20:33:00 DEBUG "metric": { 20:33:00 DEBUG "pod": "ds-cts-0" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG [...] 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG "pod": "ds-cts-2" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 1713641580, 20:33:00 DEBUG "0" 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG } 20:33:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713641580" 20:33:00 INFO [http_cmd]: http status code OK 20:33:00 DEBUG --- status code --- 20:33:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 20:33:00 DEBUG --- http response --- 20:33:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 20:33:00 DEBUG ----- output ----- 20:33:00 DEBUG { 20:33:00 DEBUG "status": "success", 20:33:00 DEBUG "data": { 20:33:00 DEBUG "resultType": "vector", 20:33:00 DEBUG "result": [ 20:33:00 DEBUG { 20:33:00 DEBUG "metric": { 20:33:00 DEBUG "instance": "10.142.0.107:9100" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG [...] 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG 20:33:00 DEBUG "instance": "10.142.0.43:9100" 20:33:00 DEBUG }, 20:33:00 DEBUG "value": [ 20:33:00 DEBUG 1713641580, 20:33:00 DEBUG "0.0006666666666660604" 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG ] 20:33:00 DEBUG } 20:33:00 DEBUG } 20:33:01 INFO 20:33:01 INFO [loop_until]: kubectl --namespace=xlou top node 20:33:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 20:33:01 INFO [loop_until]: OK (rc = 0) 20:33:01 DEBUG --- stdout --- 20:33:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6219Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 70m 0% 7614Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 57m 0% 11334Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 75m 0% 3360Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 73m 0% 14243Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5184Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 11901Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 78m 0% 2926Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 58m 0% 10842Mi 9% gke-xlou-cdm-ds-0f601751-j488 64m 0% 7227Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 77m 0% 15838Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 90m 0% 7835Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 69m 0% 13724Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 557m 3% 9408Mi 16% 20:33:01 DEBUG --- stderr --- 20:33:01 DEBUG 20:33:04 INFO Finished: True 20:33:04 INFO Waiting for threads to register finish flag 20:34:01 INFO Done. Have a nice day! :) 127.0.0.1 - - [20/Apr/2024 20:34:01] "GET /monitoring/stop HTTP/1.1" 200 - 20:34:04 WARNING File /tmp/lodemon_data-20240420_191838/node_disk_read_bytes_total.json does not exist. Skipping graph generation... WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p6481', ...) 127.0.0.1 - - [20/Apr/2024 20:34:08] "GET /monitoring/process HTTP/1.1" 200 -